Minor refactoring of WAL timelines constructor

Author: gsmolk

src/catalog.c

@@ -9,6 +9,7 @@
  */
 
 #include "pg_probackup.h"
+#include "access/timeline.h"
 
 #include <dirent.h>
 #include <signal.h>
@@ -18,12 +19,25 @@
 #include "utils/file.h"
 #include "utils/configuration.h"
 
+static pgBackup* get_closest_backup(timelineInfo *tlinfo, parray *backup_list);
+static pgBackup* get_oldest_backup(timelineInfo *tlinfo, parray *backup_list);
 static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"};
 static pgBackup *readBackupControlFile(const char *path);
 
 static bool exit_hook_registered = false;
 static parray *lock_files = NULL;
 
+static timelineInfo *
+timelineInfoNew(TimeLineID tli)
+{
+	timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo));
+	MemSet(tlinfo, 0, sizeof(timelineInfo));
+	tlinfo->tli = tli;
+	tlinfo->switchpoint = InvalidXLogRecPtr;
+	tlinfo->parent_link = NULL;
+	return tlinfo;
+}
+
 /* Iterate over locked backups and delete locks files */
 static void
 unlink_lock_atexit(void)
@@ -665,6 +679,280 @@ pgBackupCreateDir(pgBackup *backup)
 	return 0;
 }
 
+/*
+ * Create list of timelines
+ */
+parray *
+catalog_get_timelines(InstanceConfig *instance)
+{
+	parray *xlog_files_list = parray_new();
+	parray *timelineinfos;
+	parray *backups;
+	timelineInfo *tlinfo;
+	char		arclog_path[MAXPGPATH];
+
+	/* read all xlog files that belong to this archive */
+	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	parray_qsort(xlog_files_list, pgFileComparePath);
+
+	timelineinfos = parray_new();
+	tlinfo = NULL;
+
+	/* walk through files and collect info about timelines */
+	for (int i = 0; i < parray_num(xlog_files_list); i++)
+	{
+		pgFile *file = (pgFile *) parray_get(xlog_files_list, i);
+		TimeLineID tli;
+		parray *timelines;
+
+		/* regular WAL file */
+		if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN)
+		{
+			int result = 0;
+			uint32 log, seg;
+			XLogSegNo segno;
+			char suffix[MAXPGPATH];
+
+			result = sscanf(file->name, "%08X%08X%08X.%s",
+						&tli, &log, &seg, (char *) &suffix);
+
+			if (result < 3)
+			{
+				elog(WARNING, "unexpected WAL file name \"%s\"", file->name);
+				continue;
+			}
+
+			segno = log * instance->xlog_seg_size + seg;
+
+			/* regular WAL file with suffix */
+			if (result == 4)
+			{
+				/* backup history file. Currently we don't use them */
+				if (IsBackupHistoryFileName(file->name))
+				{
+					elog(VERBOSE, "backup history file \"%s\". do nothing", file->name);
+					continue;
+				}
+				/* we only expect compressed wal files with .gz suffix */
+				else if (strcmp(suffix, "gz") != 0)
+				{
+					elog(WARNING, "unexpected WAL file name \"%s\"", file->name);
+					continue;
+				}
+			}
+
+			/* new file belongs to new timeline */
+			if (!tlinfo || tlinfo->tli != tli)
+			{
+				tlinfo = timelineInfoNew(tli);
+				parray_append(timelineinfos, tlinfo);
+			}
+			/*
+			 * As it is impossible to detect if segments before segno are lost,
+			 * or just do not exist, do not report them as lost.
+			 */
+			else if (tlinfo->n_xlog_files != 0)
+			{
+				/* check, if segments are consequent */
+				XLogSegNo expected_segno = tlinfo->end_segno + 1;
+
+				/* some segments are missing. remember them in lost_segments to report */
+				if (segno != expected_segno)
+				{
+					xlogInterval *interval = palloc(sizeof(xlogInterval));;
+					interval->begin_segno = expected_segno;
+					interval->end_segno = segno - 1;
+
+					if (tlinfo->lost_segments == NULL)
+						tlinfo->lost_segments = parray_new();
+
+					parray_append(tlinfo->lost_segments, interval);
+				}
+			}
+
+			if (tlinfo->begin_segno == 0)
+				tlinfo->begin_segno = segno;
+
+			/* this file is the last for this timeline so far */
+			tlinfo->end_segno = segno;
+			/* update counters */
+			tlinfo->n_xlog_files++;
+			tlinfo->size += file->size;
+		}
+		/* timeline history file */
+		else if (IsTLHistoryFileName(file->name))
+		{
+			TimeLineHistoryEntry *tln;
+
+			sscanf(file->name, "%08X.history", &tli);
+			timelines = read_timeline_history(arclog_path, tli);
+
+			if (!tlinfo || tlinfo->tli != tli)
+			{
+				tlinfo = timelineInfoNew(tli);
+				parray_append(timelineinfos, tlinfo);
+				/*
+				 * 1 is the latest timeline in the timelines list.
+				 * 0 - is our timeline, which is of no interest here
+				 */
+				tln = (TimeLineHistoryEntry *) parray_get(timelines, 1);
+				tlinfo->switchpoint = tln->end;
+				tlinfo->parent_tli = tln->tli;
+
+				/* find parent timeline to link it with this one */
+				for (int i = 0; i < parray_num(timelineinfos); i++)
+				{
+					timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, i);
+					if (cur->tli == tlinfo->parent_tli)
+					{
+						tlinfo->parent_link = cur;
+						break;
+					}
+				}
+			}
+
+			parray_walk(timelines, pfree);
+			parray_free(timelines);
+		}
+		else
+			elog(WARNING, "unexpected WAL file name \"%s\"", file->name);
+	}
+
+	/* save information about backups belonging to each timeline */
+	backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID);
+
+	for (int i = 0; i < parray_num(timelineinfos); i++)
+	{
+		timelineInfo *tlinfo = parray_get(timelineinfos, i);
+		for (int j = 0; j < parray_num(backups); j++)
+		{
+			pgBackup *backup = parray_get(backups, j);
+			if (tlinfo->tli == backup->tli)
+			{
+				if (tlinfo->backups == NULL)
+					tlinfo->backups = parray_new();
+
+				parray_append(tlinfo->backups, backup);
+			}
+		}
+	}
+
+	/* determine oldest backup and closest backup for every timeline */
+	for (int i = 0; i < parray_num(timelineinfos); i++)
+	{
+		timelineInfo *tlinfo = parray_get(timelineinfos, i);
+
+		tlinfo->oldest_backup = get_oldest_backup(tlinfo, backups);
+		tlinfo->closest_backup = get_closest_backup(tlinfo, backups);
+	}
+
+	parray_walk(xlog_files_list, pfree);
+	parray_free(xlog_files_list);
+
+	return timelineinfos;
+}
+
+/*
+ * Iterate over parent timelines of a given timeline and look
+ * for valid backup closest to given timeline switchpoint.
+ *
+ * Returns NULL if such backup is not found.
+ */
+pgBackup*
+get_closest_backup(timelineInfo *tlinfo, parray *backup_list)
+{
+	pgBackup *closest_backup = NULL;
+	int i;
+
+	/* Only timeline with switchpoint can possibly have closest backup */
+	if (XLogRecPtrIsInvalid(tlinfo->switchpoint))
+		return NULL;
+
+	/* Only timeline with parent timeline can possibly have closest backup */
+	if (!tlinfo->parent_link)
+		return NULL;
+
+	while (tlinfo->parent_link)
+	{
+		/*
+		 * Iterate over backups belonging to parent timeline and look
+		 * for candidates.
+		 */
+		for (i = 0; i < parray_num(backup_list); i++)
+		{
+			pgBackup   *backup = parray_get(backup_list, i);
+
+			/* Backups belonging to timelines other than parent timeline can be safely skipped */
+			if (backup->tli != tlinfo->parent_tli)
+				continue;
+
+			/* Backups in future can be safely skipped */
+			if (backup->stop_lsn > tlinfo->switchpoint)
+				continue;
+
+			/* Backups with invalid STOP LSN can be safely skipped */
+			if (XLogRecPtrIsInvalid(backup->stop_lsn) ||
+				!XRecOffIsValid(backup->stop_lsn))
+				continue;
+
+			/* Only valid backups closest to switchpoint should be considered */
+			if (backup->stop_lsn <= tlinfo->switchpoint &&
+				(backup->status == BACKUP_STATUS_OK ||
+				 backup->status == BACKUP_STATUS_DONE))
+			{
+				/* Check if backup is closer to switchpoint than current candidate */
+				if (!closest_backup || backup->stop_lsn > closest_backup->stop_lsn)
+					closest_backup = backup;
+			}
+		}
+
+		/* Closest backup is found */
+		if (closest_backup)
+			break;
+
+		/* Switch to parent */
+		tlinfo = tlinfo->parent_link;
+	}
+
+	return closest_backup;
+}
+
+/*
+ * Iterate over timelines and look for oldest backup on each timeline
+ * Returns NULL if such backup is not found.
+ */
+pgBackup*
+get_oldest_backup(timelineInfo *tlinfo, parray *backup_list)
+{
+	pgBackup *oldest_backup = NULL;
+	int i;
+
+	/*
+	 * Iterate over backups belonging to timeline and look
+	 * for candidates.
+	 */
+	for (i = 0; i < parray_num(backup_list); i++)
+	{
+		pgBackup   *backup = parray_get(backup_list, i);
+
+		/* Backups belonging to other timelines can be safely skipped */
+		if (backup->tli != tlinfo->tli)
+			continue;
+
+		/* Backups with invalid START LSN can be safely skipped */
+		if (XLogRecPtrIsInvalid(backup->start_lsn) ||
+			!XRecOffIsValid(backup->start_lsn))
+			continue;
+
+		/* Check if backup is older than current candidate */
+		if (!oldest_backup || backup->start_lsn < oldest_backup->start_lsn)
+			oldest_backup = backup;
+	}
+
+	return oldest_backup;
+}
+
 /*
  * Write information about backup.in to stream "out".
  */

src/help.c

@@ -548,7 +548,7 @@ help_show(void)
 	printf(_("  -B, --backup-path=backup-path    location of the backup storage area\n"));
 	printf(_("      --instance=instance_name     show info about specific instance\n"));
 	printf(_("  -i, --backup-id=backup-id        show info about specific backups\n"));
-	printf(_("      --archive                    show WAL archive\n"));
+	printf(_("      --archive                    show WAL archive information\n"));
 	printf(_("      --format=format              show format=PLAIN|JSON\n\n"));
 }
 

src/pg_probackup.c

@@ -440,6 +440,9 @@ main(int argc, char *argv[])
 			backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD)
 			elog(ERROR, "required parameter not specified: --instance");
 	}
+	else
+		/* Set instance name */
+		instance_config.name = pgut_strdup(instance_name);
 
 	/*
 	 * If --instance option was passed, construct paths for backup data and

src/pg_probackup.h

@@ -405,8 +405,15 @@ struct timelineInfo {
 							 * about backups belonging to this timeline */
 	parray *lost_segments;	/* array of intervals of lost segments */
 	pgBackup *closest_backup; /* link to backup, closest to timeline */
+	pgBackup *oldest_backup; /* link to oldest backup on timeline */
 };
 
+typedef struct xlogInterval
+{
+	XLogSegNo begin_segno;
+	XLogSegNo end_segno;
+} xlogInterval;
+
 
 /*
  * When copying datafiles to backup we validate and compress them block
@@ -617,6 +624,7 @@ extern void catalog_lock_backup_list(parray *backup_list, int from_idx,
 extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
 											  TimeLineID tli,
 											  time_t current_start_time);
+extern parray *catalog_get_timelines(InstanceConfig *instance);
 extern void pgBackupWriteControl(FILE *out, pgBackup *backup);
 extern void write_backup_filelist(pgBackup *backup, parray *files,
 								  const char *root, parray *external_list);