-
Notifications
You must be signed in to change notification settings - Fork 86
/
Copy pathdelete.c
1124 lines (930 loc) · 32.5 KB
/
delete.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*-------------------------------------------------------------------------
*
* delete.c: delete backup files.
*
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
* Portions Copyright (c) 2015-2019, Postgres Professional
*
*-------------------------------------------------------------------------
*/
#include "pg_probackup.h"
#include <dirent.h>
#include <time.h>
#include <unistd.h>
static void delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tli,
uint32 xlog_seg_size, bool dry_run);
static void do_retention_internal(parray *backup_list, parray *to_keep_list,
parray *to_purge_list);
static void do_retention_merge(InstanceState *instanceState, parray *backup_list,
parray *to_keep_list, parray *to_purge_list,
bool no_validate, bool no_sync);
static void do_retention_purge(parray *to_keep_list, parray *to_purge_list);
static void do_retention_wal(InstanceState *instanceState, bool dry_run);
// TODO: more useful messages for dry run.
static bool backup_deleted = false; /* At least one backup was deleted */
static bool backup_merged = false; /* At least one merge was enacted */
static bool wal_deleted = false; /* At least one WAL segments was deleted */
void
do_delete(InstanceState *instanceState, time_t backup_id)
{
int i;
parray *backup_list,
*delete_list;
pgBackup *target_backup = NULL;
size_t size_to_delete = 0;
char size_to_delete_pretty[20];
/* Get complete list of backups */
backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
delete_list = parray_new();
/* Find backup to be deleted and make increment backups array to be deleted */
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
if (backup->start_time == backup_id)
{
target_backup = backup;
break;
}
}
/* sanity */
if (!target_backup)
elog(ERROR, "Failed to find backup %s, cannot delete", base36enc(backup_id));
/* form delete list */
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* check if backup is descendant of delete target */
if (is_parent(target_backup->start_time, backup, true))
{
parray_append(delete_list, backup);
elog(LOG, "Backup %s %s be deleted",
base36enc(backup->start_time), dry_run? "can":"will");
size_to_delete += backup->data_bytes;
if (backup->stream)
size_to_delete += backup->wal_bytes;
}
}
/* Report the resident size to delete */
if (size_to_delete >= 0)
{
pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty));
elog(INFO, "Resident data size to free by delete of backup %s : %s",
base36enc(target_backup->start_time), size_to_delete_pretty);
}
if (!dry_run)
{
/* Lock marked for delete backups */
catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false, true);
/* Delete backups from the end of list */
for (i = (int) parray_num(delete_list) - 1; i >= 0; i--)
{
pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i);
if (interrupted)
elog(ERROR, "interrupted during delete backup");
delete_backup_files(backup);
}
}
/* Clean WAL segments */
if (delete_wal)
do_retention_wal(instanceState, dry_run);
/* cleanup */
parray_free(delete_list);
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
}
/*
* Merge and purge backups by retention policy. Retention policy is configured by
* retention_redundancy and retention_window variables.
*
* Invalid backups handled in Oracle style, so invalid backups are ignored
* for the purpose of retention fulfillment,
* i.e. CORRUPT full backup do not taken in account when determine
* which FULL backup should be keeped for redundancy obligation(only valid do),
* but if invalid backup is not guarded by retention - it is removed
*/
void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync)
{
parray *backup_list = NULL;
parray *to_keep_list = parray_new();
parray *to_purge_list = parray_new();
bool retention_is_set = false; /* At least one retention policy is set */
bool backup_list_is_empty = false;
backup_deleted = false;
backup_merged = false;
/* For now retention is possible only locally */
MyLocation = FIO_LOCAL_HOST;
/* Get a complete list of backups. */
backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
if (parray_num(backup_list) == 0)
backup_list_is_empty = true;
if (delete_expired || merge_expired)
{
if (instance_config.retention_redundancy > 0)
elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy);
if (instance_config.retention_window > 0)
elog(LOG, "WINDOW=%u", instance_config.retention_window);
if (instance_config.retention_redundancy == 0 &&
instance_config.retention_window == 0)
{
/* Retention is disabled but we still can cleanup wal */
elog(WARNING, "Retention policy is not set");
if (!delete_wal)
return;
}
else
/* At least one retention policy is active */
retention_is_set = true;
}
if (retention_is_set && backup_list_is_empty)
elog(WARNING, "Backup list is empty, retention purge and merge are problematic");
/* Populate purge and keep lists, and show retention state messages */
if (retention_is_set && !backup_list_is_empty)
do_retention_internal(backup_list, to_keep_list, to_purge_list);
if (merge_expired && !dry_run && !backup_list_is_empty)
do_retention_merge(instanceState, backup_list, to_keep_list, to_purge_list, no_validate, no_sync);
if (delete_expired && !dry_run && !backup_list_is_empty)
do_retention_purge(to_keep_list, to_purge_list);
/* TODO: some sort of dry run for delete_wal */
if (delete_wal)
do_retention_wal(instanceState, dry_run);
/* TODO: consider dry-run flag */
if (!backup_merged)
elog(INFO, "There are no backups to merge by retention policy");
if (backup_deleted)
elog(INFO, "Purging finished");
else
elog(INFO, "There are no backups to delete by retention policy");
if (!wal_deleted)
elog(INFO, "There is no WAL to purge by retention policy");
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
parray_free(to_keep_list);
parray_free(to_purge_list);
}
/* Evaluate every backup by retention policies and populate purge and keep lists.
* Also for every backup print its status ('Active' or 'Expired') according
* to active retention policies.
*/
static void
do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list)
{
int i;
parray *redundancy_full_backup_list = NULL;
/* For retention calculation */
uint32 n_full_backups = 0;
int cur_full_backup_num = 0;
time_t days_threshold = 0;
/* For fancy reporting */
uint32 actual_window = 0;
/* Calculate n_full_backups and days_threshold */
if (instance_config.retention_redundancy > 0)
{
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
if (backup->backup_mode == BACKUP_MODE_FULL)
{
/* Add every FULL backup that satisfy Redundancy policy to separate list */
if (n_full_backups < instance_config.retention_redundancy)
{
if (!redundancy_full_backup_list)
redundancy_full_backup_list = parray_new();
parray_append(redundancy_full_backup_list, backup);
}
/* Consider only valid FULL backups for Redundancy fulfillment */
if (backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE)
{
n_full_backups++;
}
}
}
/* Sort list of full backups to keep */
if (redundancy_full_backup_list)
parray_qsort(redundancy_full_backup_list, pgBackupCompareIdDesc);
}
if (instance_config.retention_window > 0)
{
days_threshold = current_time -
(instance_config.retention_window * 60 * 60 * 24);
}
elog(INFO, "Evaluate backups by retention");
for (i = (int) parray_num(backup_list) - 1; i >= 0; i--)
{
bool redundancy_keep = false;
time_t backup_time = 0;
pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i);
/* check if backup`s FULL ancestor is in redundancy list */
if (redundancy_full_backup_list)
{
pgBackup *full_backup = find_parent_full_backup(backup);
if (full_backup && parray_bsearch(redundancy_full_backup_list,
full_backup,
pgBackupCompareIdDesc))
redundancy_keep = true;
}
/* Remember the serial number of latest valid FULL backup */
if (backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE))
{
cur_full_backup_num++;
}
/* Invalid and running backups most likely to have recovery_time == 0,
* so in this case use start_time instead.
*/
if (backup->recovery_time)
backup_time = backup->recovery_time;
else
backup_time = backup->start_time;
/* Check if backup in needed by retention policy */
if ((days_threshold == 0 || (days_threshold > backup_time)) &&
(instance_config.retention_redundancy == 0 || !redundancy_keep))
{
/* This backup is not guarded by retention
*
* Redundancy = 1
* FULL CORRUPT in retention (not count toward redundancy limit)
* FULL in retention
* ------retention redundancy -------
* PAGE3 in retention
* ------retention window -----------
* PAGE2 out of retention
* PAGE1 out of retention
* FULL out of retention <- We are here
* FULL CORRUPT out of retention
*/
/* Save backup from purge if backup is pinned and
* expire date is not yet due.
*/
if ((backup->expire_time > 0) &&
(backup->expire_time > current_time))
{
char expire_timestamp[100];
time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time, false);
elog(LOG, "Backup %s is pinned until '%s', retain",
base36enc(backup->start_time), expire_timestamp);
continue;
}
/* Add backup to purge_list */
elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time));
parray_append(to_purge_list, backup);
continue;
}
}
/* sort keep_list and purge list */
parray_qsort(to_keep_list, pgBackupCompareIdDesc);
parray_qsort(to_purge_list, pgBackupCompareIdDesc);
/* FULL
* PAGE
* PAGE <- Only such backups must go into keep list
---------retention window ----
* PAGE
* FULL
* PAGE
* FULL
*/
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
/* Do not keep invalid backups by retention
* Turns out it was not a very good idea - [Issue #114]
*/
//if (backup->status != BACKUP_STATUS_OK &&
// backup->status != BACKUP_STATUS_DONE)
// continue;
/* only incremental backups should be in keep list */
if (backup->backup_mode == BACKUP_MODE_FULL)
continue;
/* orphan backup cannot be in keep list */
if (!backup->parent_backup_link)
continue;
/* skip if backup already in purge list */
if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
continue;
/* if parent in purge_list, add backup to keep list */
if (parray_bsearch(to_purge_list,
backup->parent_backup_link,
pgBackupCompareIdDesc))
{
/* make keep list a bit more compact */
parray_append(to_keep_list, backup);
continue;
}
}
/* Message about retention state of backups
* TODO: message is ugly, rewrite it to something like show table in stdout.
*/
cur_full_backup_num = 1;
for (i = 0; i < parray_num(backup_list); i++)
{
char *action = "Active";
uint32 pinning_window = 0;
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc))
action = "Expired";
if (backup->recovery_time == 0)
actual_window = 0;
else
actual_window = (current_time - backup->recovery_time)/(3600 * 24);
/* For pinned backups show expire date */
if (backup->expire_time > 0 && backup->expire_time > backup->recovery_time)
pinning_window = (backup->expire_time - backup->recovery_time)/(3600 * 24);
/* TODO: add ancestor(chain full backup) ID */
elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s",
base36enc(backup->start_time),
pgBackupGetBackupMode(backup, false),
status2str(backup->status),
cur_full_backup_num,
instance_config.retention_redundancy,
actual_window,
pinning_window ? pinning_window : instance_config.retention_window,
action);
/* Only valid full backups are count to something */
if (backup->backup_mode == BACKUP_MODE_FULL &&
(backup->status == BACKUP_STATUS_OK ||
backup->status == BACKUP_STATUS_DONE))
cur_full_backup_num++;
}
}
/* Merge partially expired incremental chains */
static void
do_retention_merge(InstanceState *instanceState, parray *backup_list,
parray *to_keep_list, parray *to_purge_list,
bool no_validate, bool no_sync)
{
int i;
int j;
/* IMPORTANT: we can merge to only those FULL backup, that is NOT
* guarded by retention and final target of such merge must be
* an incremental backup that is guarded by retention !!!
*
* PAGE4 E
* PAGE3 D
--------retention window ---
* PAGE2 C
* PAGE1 B
* FULL A
*
* after retention merge:
* PAGE4 E
* FULL D
*/
/* Merging happens here */
for (i = 0; i < parray_num(to_keep_list); i++)
{
char *keep_backup_id = NULL;
pgBackup *full_backup = NULL;
parray *merge_list = NULL;
pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i);
/* keep list may shrink during merge */
if (!keep_backup)
continue;
elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time));
/* Got valid incremental backup, find its FULL ancestor */
full_backup = find_parent_full_backup(keep_backup);
/* Failed to find parent */
if (!full_backup)
{
elog(WARNING, "Failed to find FULL parent for %s", base36enc(keep_backup->start_time));
continue;
}
/* Check that ancestor is in purge_list */
if (!parray_bsearch(to_purge_list,
full_backup,
pgBackupCompareIdDesc))
{
elog(WARNING, "Skip backup %s for merging, "
"because his FULL parent is not marked for purge", base36enc(keep_backup->start_time));
continue;
}
/* FULL backup in purge list, thanks to compacting of keep_list current backup is
* final target for merge, but there could be intermediate incremental
* backups from purge_list.
*/
keep_backup_id = base36enc_dup(keep_backup->start_time);
elog(INFO, "Merge incremental chain between full backup %s and backup %s",
base36enc(full_backup->start_time), keep_backup_id);
pg_free(keep_backup_id);
merge_list = parray_new();
/* Form up a merge list */
while (keep_backup->parent_backup_link)
{
parray_append(merge_list, keep_backup);
keep_backup = keep_backup->parent_backup_link;
}
/* sanity */
if (!merge_list)
continue;
/* sanity */
if (parray_num(merge_list) == 0)
{
parray_free(merge_list);
continue;
}
/* In the end add FULL backup for easy locking */
parray_append(merge_list, full_backup);
/* Remove FULL backup from purge list */
parray_rm(to_purge_list, full_backup, pgBackupCompareId);
/* Lock merge chain */
catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true);
/* Consider this extreme case */
// PAGEa1 PAGEb1 both valid
// \ /
// FULL
/* Check that FULL backup do not has multiple descendants
* full_backup always point to current full_backup after merge
*/
// if (is_prolific(backup_list, full_backup))
// {
// elog(WARNING, "Backup %s has multiple valid descendants. "
// "Automatic merge is not possible.", base36enc(full_backup->start_time));
// }
/* Merge list example:
* 0 PAGE3
* 1 PAGE2
* 2 PAGE1
* 3 FULL
*
* Merge incremental chain from PAGE3 into FULL.
*/
keep_backup = parray_get(merge_list, 0);
merge_chain(instanceState, merge_list, full_backup, keep_backup, no_validate, no_sync);
backup_merged = true;
for (j = parray_num(merge_list) - 2; j >= 0; j--)
{
pgBackup *tmp_backup = (pgBackup *) parray_get(merge_list, j);
/* Try to remove merged incremental backup from both keep and purge lists */
parray_rm(to_purge_list, tmp_backup, pgBackupCompareId);
parray_set(to_keep_list, i, NULL);
}
if (!no_validate)
pgBackupValidate(full_backup, NULL);
if (full_backup->status == BACKUP_STATUS_CORRUPT)
elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time));
/* Cleanup */
parray_free(merge_list);
}
elog(INFO, "Retention merging finished");
}
/* Purge expired backups */
static void
do_retention_purge(parray *to_keep_list, parray *to_purge_list)
{
int i;
int j;
/* Remove backups by retention policy. Retention policy is configured by
* retention_redundancy and retention_window
* Remove only backups, that do not have children guarded by retention
*
* TODO: We do not consider the situation if child is marked for purge
* but parent isn`t. Maybe something bad happened with time on server?
*/
for (j = 0; j < parray_num(to_purge_list); j++)
{
bool purge = true;
pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j);
elog(LOG, "Consider backup %s for purge",
base36enc(delete_backup->start_time));
/* Evaluate marked for delete backup against every backup in keep list.
* If marked for delete backup is recognized as parent of one of those,
* then this backup should not be deleted.
*/
for (i = 0; i < parray_num(to_keep_list); i++)
{
char *keeped_backup_id;
pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i);
/* item could have been nullified in merge */
if (!keep_backup)
continue;
/* Full backup cannot be a descendant */
if (keep_backup->backup_mode == BACKUP_MODE_FULL)
continue;
keeped_backup_id = base36enc_dup(keep_backup->start_time);
elog(LOG, "Check if backup %s is parent of backup %s",
base36enc(delete_backup->start_time), keeped_backup_id);
if (is_parent(delete_backup->start_time, keep_backup, true))
{
/* We must not delete this backup, evict it from purge list */
elog(LOG, "Retain backup %s because his "
"descendant %s is guarded by retention",
base36enc(delete_backup->start_time), keeped_backup_id);
purge = false;
pg_free(keeped_backup_id);
break;
}
pg_free(keeped_backup_id);
}
/* Retain backup */
if (!purge)
continue;
/* Actual purge */
if (!lock_backup(delete_backup, false, true))
{
/* If the backup still is used, do not interrupt and go to the next */
elog(WARNING, "Cannot lock backup %s directory, skip purging",
base36enc(delete_backup->start_time));
continue;
}
/* Delete backup and update status to DELETED */
delete_backup_files(delete_backup);
backup_deleted = true;
}
}
/*
* Purge WAL
* Iterate over timelines
* Look for WAL segment not reachable from existing backups
* and delete them.
*/
static void
do_retention_wal(InstanceState *instanceState, bool dry_run)
{
parray *tli_list;
int i;
//TODO check that instanceState is not NULL
tli_list = catalog_get_timelines(instanceState, &instance_config);
for (i = 0; i < parray_num(tli_list); i++)
{
timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i);
/*
* Empty timeline (only mentioned in timeline history file)
* has nothing to cleanup.
*/
if (tlinfo->n_xlog_files == 0 && parray_num(tlinfo->xlog_filelist) == 0)
continue;
/*
* If closest backup exists, then timeline is reachable from
* at least one backup and no file should be removed.
* Unless wal-depth is enabled.
*/
if ((tlinfo->closest_backup) && instance_config.wal_depth <= 0)
continue;
/* WAL retention keeps this timeline from purge */
if (instance_config.wal_depth >= 0 && tlinfo->anchor_tli > 0 &&
tlinfo->anchor_tli != tlinfo->tli)
continue;
/*
* Purge all WAL segments before START LSN of oldest backup.
* If timeline doesn't have a backup, then whole timeline
* can be safely purged.
* Note, that oldest_backup is not necessarily valid here,
* but still we keep wal for it.
* If wal-depth is enabled then use anchor_lsn instead
* of oldest_backup.
*/
if (tlinfo->oldest_backup)
{
if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn)))
{
delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn,
tlinfo, instance_config.xlog_seg_size, dry_run);
}
else
{
delete_walfiles_in_tli(instanceState, tlinfo->oldest_backup->start_lsn,
tlinfo, instance_config.xlog_seg_size, dry_run);
}
}
else
{
if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn)))
delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn,
tlinfo, instance_config.xlog_seg_size, dry_run);
else
delete_walfiles_in_tli(instanceState, InvalidXLogRecPtr,
tlinfo, instance_config.xlog_seg_size, dry_run);
}
}
}
/*
* Delete backup files of the backup and update the status of the backup to
* BACKUP_STATUS_DELETED.
* TODO: delete files on multiple threads
*/
void
delete_backup_files(pgBackup *backup)
{
size_t i;
char timestamp[100];
parray *files;
size_t num_files;
char full_path[MAXPGPATH];
/*
* If the backup was deleted already, there is nothing to do.
*/
if (backup->status == BACKUP_STATUS_DELETED)
{
elog(WARNING, "Backup %s already deleted",
base36enc(backup->start_time));
return;
}
if (backup->recovery_time)
time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false);
else
time2iso(timestamp, lengthof(timestamp), backup->start_time, false);
elog(INFO, "Delete: %s %s",
base36enc(backup->start_time), timestamp);
/*
* Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which
* the error occurs before deleting all backup files.
*/
write_backup_status(backup, BACKUP_STATUS_DELETING, false);
/* list files to be deleted */
files = parray_new();
dir_list_file(files, backup->root_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST);
/* delete leaf node first */
parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
num_files = parray_num(files);
for (i = 0; i < num_files; i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
join_path_components(full_path, backup->root_dir, file->rel_path);
if (interrupted)
elog(ERROR, "interrupted during delete backup");
if (progress)
elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"",
i + 1, num_files, full_path);
pgFileDelete(file->mode, full_path);
}
parray_walk(files, pgFileFree);
parray_free(files);
backup->status = BACKUP_STATUS_DELETED;
return;
}
/*
* Purge WAL archive. One timeline at a time.
* If 'keep_lsn' is InvalidXLogRecPtr, then whole timeline can be purged
* If 'keep_lsn' is valid LSN, then every lesser segment can be purged.
* If 'dry_run' is set, then don`t actually delete anything.
*
* Case 1:
* archive is not empty, 'keep_lsn' is valid and we can delete something.
* Case 2:
* archive is not empty, 'keep_lsn' is valid and prevening us from deleting anything.
* Case 3:
* archive is not empty, 'keep_lsn' is invalid, drop all WAL files in archive,
* belonging to the timeline.
* Case 4:
* archive is empty, 'keep_lsn' is valid, assume corruption of WAL archive.
* Case 5:
* archive is empty, 'keep_lsn' is invalid, drop backup history files
* and partial WAL segments in archive.
*
* Q: Maybe we should stop treating partial WAL segments as second-class citizens?
*/
static void
delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tlinfo,
uint32 xlog_seg_size, bool dry_run)
{
XLogSegNo FirstToDeleteSegNo;
XLogSegNo OldestToKeepSegNo = 0;
char first_to_del_str[MAXFNAMELEN];
char oldest_to_keep_str[MAXFNAMELEN];
int i;
size_t wal_size_logical = 0;
size_t wal_size_actual = 0;
char wal_pretty_size[20];
bool purge_all = false;
/* Timeline is completely empty */
if (parray_num(tlinfo->xlog_filelist) == 0)
{
elog(INFO, "Timeline %i is empty, nothing to remove", tlinfo->tli);
return;
}
if (XLogRecPtrIsInvalid(keep_lsn))
{
/* Drop all files in timeline */
elog(INFO, "On timeline %i all files %s be removed",
tlinfo->tli, dry_run?"can":"will");
FirstToDeleteSegNo = tlinfo->begin_segno;
OldestToKeepSegNo = tlinfo->end_segno;
purge_all = true;
}
else
{
/* Drop all segments between begin_segno and segment with keep_lsn (excluding) */
FirstToDeleteSegNo = tlinfo->begin_segno;
GetXLogSegNo(keep_lsn, OldestToKeepSegNo, xlog_seg_size);
}
if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo)
{
/* translate segno number into human readable format */
GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size);
GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size);
elog(INFO, "On timeline %i WAL segments between %s and %s %s be removed",
tlinfo->tli, first_to_del_str,
oldest_to_keep_str, dry_run?"can":"will");
}
/* sanity */
if (OldestToKeepSegNo > FirstToDeleteSegNo)
{
wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size;
/* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */
if (purge_all)
wal_size_logical += xlog_seg_size;
}
else if (OldestToKeepSegNo < FirstToDeleteSegNo)
{
/* It is actually possible for OldestToKeepSegNo to be less than FirstToDeleteSegNo
* in case of :
* 1. WAL archive corruption.
* 2. There is no actual WAL archive to speak of and
* 'keep_lsn' is coming from STREAM backup.
*/
if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0)
{
GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size);
GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size);
elog(LOG, "On timeline %i first segment %s is greater than oldest segment to keep %s",
tlinfo->tli, first_to_del_str, oldest_to_keep_str);
}
}
else if (OldestToKeepSegNo == FirstToDeleteSegNo && !purge_all)
{
/* 'Nothing to delete' scenario because of 'keep_lsn'
* with possible exception of partial and backup history files.
*/
elog(INFO, "Nothing to remove on timeline %i", tlinfo->tli);
}
/* Report the logical size to delete */
if (wal_size_logical > 0)
{
pretty_size(wal_size_logical, wal_pretty_size, lengthof(wal_pretty_size));
elog(INFO, "Logical WAL size to remove on timeline %i : %s",
tlinfo->tli, wal_pretty_size);
}
/* Calculate the actual size to delete */
for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++)
{
xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i);
if (purge_all || wal_file->segno < OldestToKeepSegNo)
wal_size_actual += wal_file->file.size;
}
/* Report the actual size to delete */
if (wal_size_actual > 0)
{
pretty_size(wal_size_actual, wal_pretty_size, lengthof(wal_pretty_size));
elog(INFO, "Resident WAL size to free on timeline %i : %s",
tlinfo->tli, wal_pretty_size);
}
if (dry_run)
return;
for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++)
{
xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i);
if (interrupted)
elog(ERROR, "interrupted during WAL archive purge");
/* Any segment equal or greater than EndSegNo must be kept
* unless it`s a 'purge all' scenario.
*/
if (purge_all || wal_file->segno < OldestToKeepSegNo)
{
char wal_fullpath[MAXPGPATH];
join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name);
/* save segment from purging */
if (instance_config.wal_depth >= 0 && wal_file->keep)
{
elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath);
continue;
}
/* unlink segment */
if (fio_unlink(wal_fullpath, FIO_BACKUP_HOST) < 0)
{
/* Missing file is not considered as error condition */
if (errno != ENOENT)
elog(ERROR, "Could not remove file \"%s\": %s",
wal_fullpath, strerror(errno));
}
else
{
if (wal_file->type == SEGMENT)
elog(VERBOSE, "Removed WAL segment \"%s\"", wal_fullpath);
else if (wal_file->type == TEMP_SEGMENT)
elog(VERBOSE, "Removed temp WAL segment \"%s\"", wal_fullpath);
else if (wal_file->type == PARTIAL_SEGMENT)
elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_fullpath);
else if (wal_file->type == BACKUP_HISTORY_FILE)
elog(VERBOSE, "Removed backup history file \"%s\"", wal_fullpath);
}
wal_deleted = true;
}
}
}
/* Delete all backup files and wal files of given instance. */
int
do_delete_instance(InstanceState *instanceState)
{
parray *backup_list;
int i;
/* Delete all backups. */
backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID);
catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true);
for (i = 0; i < parray_num(backup_list); i++)
{
pgBackup *backup = (pgBackup *) parray_get(backup_list, i);
delete_backup_files(backup);
}
/* Cleanup */
parray_walk(backup_list, pgBackupFree);
parray_free(backup_list);
/* Delete all wal files. */
pgut_rmtree(instanceState->instance_wal_subdir_path, false, true);