Skip to content

Commit d34a6a3

Browse files
committed
[Issue #116] WAL archive information
1 parent 11ab44f commit d34a6a3

25 files changed

+1945
-366
lines changed

Documentation.md

+272-5
Large diffs are not rendered by default.

src/archive.c

+17-17
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
#include <unistd.h>
1414

1515
static void push_wal_file(const char *from_path, const char *to_path,
16-
bool is_compress, bool overwrite);
16+
bool is_compress, bool overwrite, int compress_level);
1717
static void get_wal_file(const char *from_path, const char *to_path);
1818
#ifdef HAVE_LIBZ
1919
static const char *get_gz_error(gzFile gzf, int errnum);
@@ -31,11 +31,10 @@ static void copy_file_attributes(const char *from_path,
3131
* --wal-file-path %p --wal-file-name %f', to move backups into arclog_path.
3232
* Where archlog_path is $BACKUP_PATH/wal/system_id.
3333
* Currently it just copies wal files to the new location.
34-
* TODO: Planned options: list the arclog content,
35-
* compute and validate checksums.
3634
*/
3735
int
38-
do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite)
36+
do_archive_push(InstanceConfig *instance,
37+
char *wal_file_path, char *wal_file_name, bool overwrite)
3938
{
4039
char backup_wal_file_path[MAXPGPATH];
4140
char absolute_wal_file_path[MAXPGPATH];
@@ -60,33 +59,33 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite)
6059
/* verify that archive-push --instance parameter is valid */
6160
system_id = get_system_identifier(current_dir);
6261

63-
if (instance_config.pgdata == NULL)
62+
if (instance->pgdata == NULL)
6463
elog(ERROR, "cannot read pg_probackup.conf for this instance");
6564

66-
if(system_id != instance_config.system_identifier)
65+
if(system_id != instance->system_identifier)
6766
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
6867
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
69-
wal_file_name, instance_name, instance_config.system_identifier,
68+
wal_file_name, instance->name, instance->system_identifier,
7069
system_id);
7170

7271
/* Create 'archlog_path' directory. Do nothing if it already exists. */
73-
fio_mkdir(arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST);
72+
fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST);
7473

7574
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
76-
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
75+
join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name);
7776

7877
elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path);
7978

80-
if (instance_config.compress_alg == PGLZ_COMPRESS)
79+
if (instance->compress_alg == PGLZ_COMPRESS)
8180
elog(ERROR, "pglz compression is not supported");
8281

8382
#ifdef HAVE_LIBZ
84-
if (instance_config.compress_alg == ZLIB_COMPRESS)
83+
if (instance->compress_alg == ZLIB_COMPRESS)
8584
is_compress = IsXLogFileName(wal_file_name);
8685
#endif
8786

8887
push_wal_file(absolute_wal_file_path, backup_wal_file_path, is_compress,
89-
overwrite);
88+
overwrite, instance->compress_level);
9089
elog(INFO, "pg_probackup archive-push completed successfully");
9190

9291
return 0;
@@ -97,7 +96,8 @@ do_archive_push(char *wal_file_path, char *wal_file_name, bool overwrite)
9796
* Move files from arclog_path to pgdata/wal_file_path.
9897
*/
9998
int
100-
do_archive_get(char *wal_file_path, char *wal_file_name)
99+
do_archive_get(InstanceConfig *instance,
100+
char *wal_file_path, char *wal_file_name)
101101
{
102102
char backup_wal_file_path[MAXPGPATH];
103103
char absolute_wal_file_path[MAXPGPATH];
@@ -118,7 +118,7 @@ do_archive_get(char *wal_file_path, char *wal_file_name)
118118
elog(ERROR, "getcwd() error");
119119

120120
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
121-
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
121+
join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name);
122122

123123
elog(INFO, "pg_probackup archive-get from %s to %s",
124124
backup_wal_file_path, absolute_wal_file_path);
@@ -134,7 +134,7 @@ do_archive_get(char *wal_file_path, char *wal_file_name)
134134
*/
135135
void
136136
push_wal_file(const char *from_path, const char *to_path, bool is_compress,
137-
bool overwrite)
137+
bool overwrite, int compress_level)
138138
{
139139
FILE *in = NULL;
140140
int out = -1;
@@ -183,7 +183,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
183183
{
184184
snprintf(to_path_temp, sizeof(to_path_temp), "%s.part", gz_to_path);
185185

186-
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST);
186+
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST);
187187
if (gz_out == NULL)
188188
{
189189
partial_file_exists = true;
@@ -246,7 +246,7 @@ push_wal_file(const char *from_path, const char *to_path, bool is_compress,
246246
#ifdef HAVE_LIBZ
247247
if (is_compress)
248248
{
249-
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, instance_config.compress_level, FIO_BACKUP_HOST);
249+
gz_out = fio_gzopen(to_path_temp, PG_BINARY_W, compress_level, FIO_BACKUP_HOST);
250250
if (gz_out == NULL)
251251
elog(ERROR, "Cannot open destination temporary WAL file \"%s\": %s",
252252
to_path_temp, strerror(errno));

src/backup.c

+4-2
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo)
196196
char prev_backup_filelist_path[MAXPGPATH];
197197

198198
/* get list of backups already taken */
199-
backup_list = catalog_get_backup_list(INVALID_BACKUP_ID);
199+
backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID);
200200

201201
prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time);
202202
if (prev_backup == NULL)
@@ -1681,7 +1681,9 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
16811681
* In case of backup from replica >= 9.6 we do not trust minRecPoint
16821682
* and stop_backup LSN, so we use latest replayed LSN as STOP LSN.
16831683
*/
1684-
if (backup->from_replica)
1684+
1685+
/* current is used here because of cleanup */
1686+
if (current.from_replica)
16851687
stop_backup_query = "SELECT"
16861688
" pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot()),"
16871689
" current_timestamp(0)::timestamptz,"

src/catalog.c

+102-5
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,14 @@ unlink_lock_atexit(void)
5353
* If no backup matches, return NULL.
5454
*/
5555
pgBackup *
56-
read_backup(time_t timestamp)
56+
read_backup(const char *instance_name, time_t timestamp)
5757
{
5858
pgBackup tmp;
5959
char conf_path[MAXPGPATH];
6060

6161
tmp.start_time = timestamp;
62-
pgBackupGetPath(&tmp, conf_path, lengthof(conf_path), BACKUP_CONTROL_FILE);
62+
pgBackupGetPathInInstance(instance_name, &tmp, conf_path,
63+
lengthof(conf_path), BACKUP_CONTROL_FILE, NULL);
6364

6465
return readBackupControlFile(conf_path);
6566
}
@@ -71,11 +72,12 @@ read_backup(time_t timestamp)
7172
* status.
7273
*/
7374
void
74-
write_backup_status(pgBackup *backup, BackupStatus status)
75+
write_backup_status(pgBackup *backup, BackupStatus status,
76+
const char *instance_name)
7577
{
7678
pgBackup *tmp;
7779

78-
tmp = read_backup(backup->start_time);
80+
tmp = read_backup(instance_name, backup->start_time);
7981
if (!tmp)
8082
{
8183
/*
@@ -302,19 +304,86 @@ IsDir(const char *dirpath, const char *entry, fio_location location)
302304
return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode);
303305
}
304306

307+
/*
308+
* Create list of instances in given backup catalog.
309+
*
310+
* Returns parray of "InstanceConfig" structures, filled with
311+
* actual config of each instance.
312+
*/
313+
parray *
314+
catalog_get_instance_list(void)
315+
{
316+
char path[MAXPGPATH];
317+
DIR *dir;
318+
struct dirent *dent;
319+
parray *instances;
320+
321+
instances = parray_new();
322+
323+
/* open directory and list contents */
324+
join_path_components(path, backup_path, BACKUPS_DIR);
325+
dir = opendir(path);
326+
if (dir == NULL)
327+
elog(ERROR, "Cannot open directory \"%s\": %s",
328+
path, strerror(errno));
329+
330+
while (errno = 0, (dent = readdir(dir)) != NULL)
331+
{
332+
char child[MAXPGPATH];
333+
struct stat st;
334+
InstanceConfig *instance;
335+
336+
/* skip entries point current dir or parent dir */
337+
if (strcmp(dent->d_name, ".") == 0 ||
338+
strcmp(dent->d_name, "..") == 0)
339+
continue;
340+
341+
join_path_components(child, path, dent->d_name);
342+
343+
if (lstat(child, &st) == -1)
344+
elog(ERROR, "Cannot stat file \"%s\": %s",
345+
child, strerror(errno));
346+
347+
if (!S_ISDIR(st.st_mode))
348+
continue;
349+
350+
instance = readInstanceConfigFile(dent->d_name);
351+
352+
parray_append(instances, instance);
353+
}
354+
355+
/* TODO 3.0: switch to ERROR */
356+
if (parray_num(instances) == 0)
357+
elog(WARNING, "This backup catalog contains no backup instances. Backup instance can be added via 'add-instance' command.");
358+
359+
if (errno)
360+
elog(ERROR, "Cannot read directory \"%s\": %s",
361+
path, strerror(errno));
362+
363+
if (closedir(dir))
364+
elog(ERROR, "Cannot close directory \"%s\": %s",
365+
path, strerror(errno));
366+
367+
return instances;
368+
}
369+
305370
/*
306371
* Create list of backups.
307372
* If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups.
308373
* The list is sorted in order of descending start time.
309374
* If valid backup id is passed only matching backup will be added to the list.
310375
*/
311376
parray *
312-
catalog_get_backup_list(time_t requested_backup_id)
377+
catalog_get_backup_list(const char *instance_name, time_t requested_backup_id)
313378
{
314379
DIR *data_dir = NULL;
315380
struct dirent *data_ent = NULL;
316381
parray *backups = NULL;
317382
int i;
383+
char backup_instance_path[MAXPGPATH];
384+
385+
sprintf(backup_instance_path, "%s/%s/%s",
386+
backup_path, BACKUPS_DIR, instance_name);
318387

319388
/* open backup instance backups directory */
320389
data_dir = fio_opendir(backup_instance_path, FIO_BACKUP_HOST);
@@ -420,6 +489,7 @@ catalog_get_backup_list(time_t requested_backup_id)
420489
* Create list of backup datafiles.
421490
* If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error.
422491
* If valid backup id is passed only matching backup will be added to the list.
492+
* TODO this function only used once. Is it really needed?
423493
*/
424494
parray *
425495
get_backup_filelist(pgBackup *backup)
@@ -1195,6 +1265,33 @@ pgBackupGetPath2(const pgBackup *backup, char *path, size_t len,
11951265
base36enc(backup->start_time), subdir1, subdir2);
11961266
}
11971267

1268+
/*
1269+
* independent from global variable backup_instance_path
1270+
* Still depends from backup_path
1271+
*/
1272+
void
1273+
pgBackupGetPathInInstance(const char *instance_name,
1274+
const pgBackup *backup, char *path, size_t len,
1275+
const char *subdir1, const char *subdir2)
1276+
{
1277+
char backup_instance_path[MAXPGPATH];
1278+
1279+
sprintf(backup_instance_path, "%s/%s/%s",
1280+
backup_path, BACKUPS_DIR, instance_name);
1281+
1282+
/* If "subdir1" is NULL do not check "subdir2" */
1283+
if (!subdir1)
1284+
snprintf(path, len, "%s/%s", backup_instance_path,
1285+
base36enc(backup->start_time));
1286+
else if (!subdir2)
1287+
snprintf(path, len, "%s/%s/%s", backup_instance_path,
1288+
base36enc(backup->start_time), subdir1);
1289+
/* "subdir1" and "subdir2" is not NULL */
1290+
else
1291+
snprintf(path, len, "%s/%s/%s/%s", backup_instance_path,
1292+
base36enc(backup->start_time), subdir1, subdir2);
1293+
}
1294+
11981295
/*
11991296
* Check if multiple backups consider target backup to be their direct parent
12001297
*/

0 commit comments

Comments
 (0)