[Issue #68] the backup pinning

Author: gsmolk

Documentation.md

@@ -38,6 +38,7 @@ Current version - 2.1.5
     * [Managing the Backup Catalog](#managing-the-backup-catalog)
         * [Viewing WAL Archive Information](#viewing-wal-archive-information)
     * [Configuring Backup Retention Policy](#configuring-backup-retention-policy)
+        * [Pinning a Backup]
     * [Merging Backups](#merging-backups)
     * [Deleting Backups](#deleting-backups)
 
@@ -49,6 +50,7 @@ Current version - 2.1.5
         * [add-instance](#add-instance)
         * [del-instance](#del-instance)
         * [set-config](#set-config)
+        * [set-backup](#set-backup)
         * [show-config](#show-config)
         * [show](#show)
         * [backup](#backup)
@@ -63,6 +65,7 @@ Current version - 2.1.5
         * [Common Options](#common-options)
         * [Recovery Target Options](#recovery-target-options)
         * [Retention Options](#retention-options)
+            * [Pinning Options](#pinning-options)
         * [Logging Options](#logging-options)
         * [Connection Options](#connection-options)
         * [Compression Options](#compression-options)
@@ -89,6 +92,8 @@ Current version - 2.1.5
 
 `pg_probackup set-config -B backup_dir --instance instance_name [option...]`
 
+`pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id [option...]`
+
 `pg_probackup show-config -B backup_dir --instance instance_name [--format=format]`
 
 `pg_probackup show -B backup_dir [option...]`
@@ -775,6 +780,7 @@ start-time = '2017-05-16 12:57:29'
 end-time = '2017-05-16 12:57:31'
 recovery-xid = 597
 recovery-time = '2017-05-16 12:57:31'
+expire-time = '2020-05-16 12:57:31'
 data-bytes = 22288792
 wal-bytes = 16777216
 uncompressed-bytes = 39961833
@@ -794,6 +800,7 @@ Detailed output has additional attributes:
 - program-version — full version of pg_probackup binary used to create backup.
 - start-time — the backup starting time.
 - end-time — the backup ending time.
+- expire-time — if the backup was pinned, then until this point in time the backup cannot be removed by retention purge.
 - uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used.
 - pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'.
 - recovery-xid — current transaction id at the moment of backup ending.
@@ -1171,6 +1178,42 @@ BACKUP INSTANCE 'node'
 
 >NOTE: The Time field for the merged backup displays the time required for the merge.
 
+#### Pinning a Backup
+
+If you have the necessity to exclude certain backups from established retention policy then it is possible to pin a backup for an arbitrary amount of time. Example:
+
+    pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d
+
+This command will set `expire-time` of specified backup to 30 days starting from backup `recovery-time` attribute. Basically 'expire-time = recovery-time + ttl'.
+
+You can set `expire-time` explicitly using `--expire-time` option. Example:
+
+    pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03'
+
+Alternatively you can use the `--ttl` and `--expire-time` options with the [backup](#backup) command to pin newly created backup:
+
+    pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d
+    pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03'
+
+You can determine the fact that backup is pinned and check due expire time by looking up 'expire-time' attribute in backup metadata via (show)[#show] command:
+
+    pg_probackup show --instance instance_name -i backup_id
+
+Pinned backup has `expire-time` attribute:
+```
+...
+recovery-time = '2017-05-16 12:57:31'
+expire-time = '2020-01-01 00:00:00+03'
+data-bytes = 22288792
+...
+```
+
+You can unpin a backup by setting `--ttl` option to zero using `set-backup` command. Example:
+
+    pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0
+
+Only pinned backups have `expire-time` attribute in backup metadata.
+
 ### Merging Backups
 
 As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge:
@@ -1267,6 +1310,15 @@ For all available settings, see the [Options](#options) section.
 
 It is **not recommended** to edit pg_probackup.conf manually.
 
+#### set-backup
+
+    pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id
+    {--ttl=ttl | --expire-time=time} [--help]
+
+Sets the provided backup-specific settings into the backup.control configuration file, or modifies previously defined values.
+
+For all available settings, see the section [Pinning Options](#pinning-options).
+
 #### show-config
 
     pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json]
@@ -1296,7 +1348,7 @@ For details on usage, see the sections [Managing the Backup Catalog](#managing-t
     [-w --no-password] [-W --password]
     [--archive-timeout=timeout] [--external-dirs=external_directory_path]
     [connection_options] [compression_options] [remote_options]
-    [retention_options] [logging_options]
+    [retention_options] [pinning_options] [logging_options]
 
 Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use.
 
@@ -1342,7 +1394,7 @@ Disables block-level checksum verification to speed up backup.
     --no-validate
 Skips automatic validation after successfull backup. You can use this flag if you validate backups regularly and would like to save time when running backup operations.
 
-Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used.
+Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Pinning Options](#pinning-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used.
 
 For details on usage, see the section [Creating a Backup](#creating-a-backup).
 
@@ -1565,6 +1617,18 @@ Merges the oldest incremental backup that satisfies the requirements of retentio
     --dry-run
 Displays the current status of all the available backups, without deleting or merging expired backups, if any.
 
+##### Pinning Options
+
+You can use these options together with [backup](#backup) and [set-delete](#set-backup) commands.
+
+For details on backup pinning, see the section [Pinning a Backup](#pinning-a-backup).
+
+    --ttl=ttl
+Specifies the amount of time the backup should be pinned. Must be a positive integer. The zero value unpin already pinned backup. Supported units: ms, s, min, h, d (s by default). Example: `--ttl=30d`.
+
+    --expire-time=time
+Specifies the timestamp up to which the backup will stay pinned. Must be a ISO-8601 complaint timestamp. Example: `--expire-time='2020-01-01 00:00:00+03'`
+
 #### Logging Options
 
 You can use these options with any command.

src/backup.c

@@ -695,7 +695,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
  * Entry point of pg_probackup BACKUP subcommand.
  */
 int
-do_backup(time_t start_time, bool no_validate)
+do_backup(time_t start_time, bool no_validate,
+			pgSetBackupParams *set_backup_params)
 {
 	PGconn *backup_conn = NULL;
 	PGNodeInfo nodeInfo;
@@ -801,6 +802,15 @@ do_backup(time_t start_time, bool no_validate)
 	current.status = BACKUP_STATUS_DONE;
 	write_backup(&current);
 
+	/* Pin backup if requested */
+	if (set_backup_params &&
+		(set_backup_params->ttl > 0 ||
+		 set_backup_params->expire_time > 0))
+	{
+		if (!pin_backup(&current, set_backup_params))
+			elog(ERROR, "Failed to pin the backup %s", base36enc(current.backup_id));
+	}
+
 	if (!no_validate)
 		pgBackupValidate(&current, NULL);
 

src/catalog.c

@@ -987,6 +987,81 @@ get_oldest_backup(timelineInfo *tlinfo)
 	return oldest_backup;
 }
 
+/*
+ * Overwrite backup metadata.
+ */
+void
+do_set_backup(const char *instance_name, time_t backup_id,
+			  pgSetBackupParams *set_backup_params)
+{
+	pgBackup	*target_backup = NULL;
+	parray 		*backup_list = NULL;
+
+	if (!set_backup_params)
+		elog(ERROR, "Nothing to set by 'set-backup' command");
+
+	backup_list = catalog_get_backup_list(instance_name, backup_id);
+	if (parray_num(backup_list) != 1)
+		elog(ERROR, "Failed to find backup %s", base36enc(backup_id));
+
+	target_backup = (pgBackup *) parray_get(backup_list, 0);
+
+	if (!pin_backup(target_backup, set_backup_params))
+		elog(ERROR, "Failed to pin the backup %s", base36enc(backup_id));
+}
+
+/*
+ * Set 'expire-time' attribute based on set_backup_params, or unpin backup
+ * if ttl is equal to zero.
+ */
+bool
+pin_backup(pgBackup	*target_backup, pgSetBackupParams *set_backup_params)
+{
+
+	if (target_backup->recovery_time <= 0)
+		elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'",
+						base36enc(target_backup->backup_id));
+
+	/* Pin comes from ttl */
+	if (set_backup_params->ttl > 0)
+		target_backup->expire_time = target_backup->recovery_time + set_backup_params->ttl;
+	/* Unpin backup */
+	else if (set_backup_params->ttl == 0)
+	{
+		/* If backup was not pinned in the first place,
+		 * then there is nothing to unpin.
+		 */
+		if (target_backup->expire_time == 0)
+		{
+			elog(WARNING, "Backup %s is not pinned, nothing to unpin",
+									base36enc(target_backup->start_time));
+			return false;
+		}
+		target_backup->expire_time = 0;
+	}
+	/* Pin comes from expire-time */
+	else if (set_backup_params->expire_time > 0)
+		target_backup->expire_time = set_backup_params->expire_time;
+	else
+		return false;
+
+	/* Update backup.control */
+	write_backup(target_backup);
+
+	if (set_backup_params->ttl > 0 || set_backup_params->expire_time > 0)
+	{
+		char	expire_timestamp[100];
+
+		time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time);
+		elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time),
+														expire_timestamp);
+	}
+	else
+		elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time));
+
+	return true;
+}
+
 /*
  * Write information about backup.in to stream "out".
  */
@@ -1041,6 +1116,11 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 		time2iso(timestamp, lengthof(timestamp), backup->recovery_time);
 		fio_fprintf(out, "recovery-time = '%s'\n", timestamp);
 	}
+	if (backup->expire_time > 0)
+	{
+		time2iso(timestamp, lengthof(timestamp), backup->expire_time);
+		fio_fprintf(out, "expire-time = '%s'\n", timestamp);
+	}
 
 	/*
 	 * Size of PGDATA directory. The size does not include size of related
@@ -1284,6 +1364,7 @@ readBackupControlFile(const char *path)
 		{'t', 0, "end-time",			&backup->end_time, SOURCE_FILE_STRICT},
 		{'U', 0, "recovery-xid",		&backup->recovery_xid, SOURCE_FILE_STRICT},
 		{'t', 0, "recovery-time",		&backup->recovery_time, SOURCE_FILE_STRICT},
+		{'t', 0, "expire-time",			&backup->expire_time, SOURCE_FILE_STRICT},
 		{'I', 0, "data-bytes",			&backup->data_bytes, SOURCE_FILE_STRICT},
 		{'I', 0, "wal-bytes",			&backup->wal_bytes, SOURCE_FILE_STRICT},
 		{'I', 0, "uncompressed-bytes",	&backup->uncompressed_bytes, SOURCE_FILE_STRICT},
@@ -1529,6 +1610,7 @@ pgBackupInit(pgBackup *backup)
 	backup->end_time = (time_t) 0;
 	backup->recovery_xid = 0;
 	backup->recovery_time = (time_t) 0;
+	backup->expire_time = (time_t) 0;
 
 	backup->data_bytes = BYTES_INVALID;
 	backup->wal_bytes = BYTES_INVALID;

src/delete.c

@@ -292,6 +292,20 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 			 * FULL CORRUPT out of retention
 			 */
 
+			/* Save backup from purge if backup is pinned and
+			 * expire date is not yet due.
+			 */
+			if ((backup->expire_time > 0) &&
+				(backup->expire_time > current_time))
+			{
+				char		expire_timestamp[100];
+				time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time);
+
+				elog(LOG, "Backup %s is pinned until '%s', retain",
+					base36enc(backup->start_time), expire_timestamp);
+				continue;
+			}
+
 			/* Add backup to purge_list */
 			elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time));
 			parray_append(to_purge_list, backup);
@@ -355,6 +369,7 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 	for (i = 0; i < parray_num(backup_list); i++)
 	{
 		char		*action = "Active";
+		uint32		pinning_window = 0;
 
 		pgBackup	*backup = (pgBackup *) parray_get(backup_list, i);
 
@@ -364,7 +379,11 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 		if (backup->recovery_time == 0)
 			actual_window = 0;
 		else
-			actual_window = (current_time - backup->recovery_time)/(60 * 60 * 24);
+			actual_window = (current_time - backup->recovery_time)/(3600 * 24);
+
+		/* For pinned backups show expire date */
+		if (backup->expire_time > 0 && backup->expire_time > backup->recovery_time)
+			pinning_window = (backup->expire_time - backup->recovery_time)/(3600 * 24);
 
 		/* TODO: add ancestor(chain full backup) ID */
 		elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s",
@@ -373,7 +392,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg
 				status2str(backup->status),
 				cur_full_backup_num,
 				instance_config.retention_redundancy,
-				actual_window, instance_config.retention_window,
+				actual_window,
+				pinning_window ? pinning_window : instance_config.retention_window,
 				action);
 
 		if (backup->backup_mode == BACKUP_MODE_FULL)
@@ -576,7 +596,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list)
 			{
 
 				/* We must not delete this backup, evict it from purge list */
-				elog(LOG, "Retain backup %s from purge because his "
+				elog(LOG, "Retain backup %s because his "
 					"descendant %s is guarded by retention",
 						base36enc(delete_backup->start_time), keeped_backup_id);