Merge branch 'release_2_4' into issue_66

Author: gsmolk

src/backup.c

@@ -83,7 +83,7 @@ static void *backup_files(void *arg);
 static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs);
 
 static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
-							PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn);
+							PGNodeInfo *nodeInfo, PGconn *conn);
 static void pg_switch_wal(PGconn *conn);
 static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo);
 static int checkpoint_timeout(PGconn *backup_conn);
@@ -149,9 +149,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 	parray	   *external_dirs = NULL;
 	parray	   *database_map = NULL;
 
-	PGconn	   *master_conn = NULL;
-	PGconn	   *pg_startbackup_conn = NULL;
-
 	/* used for multitimeline incremental backup */
 	parray       *tli_list = NULL;
 
@@ -168,13 +165,34 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 		check_external_for_tablespaces(external_dirs, backup_conn);
 	}
 
+	/* Clear ptrack files for not PTRACK backups */
+	if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable)
+		pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num);
+
+	/* notify start of backup to PostgreSQL server */
+	time2iso(label, lengthof(label), current.start_time);
+	strncat(label, " with pg_probackup", lengthof(label) -
+			strlen(" with pg_probackup"));
+
+	/* Call pg_start_backup function in PostgreSQL connect */
+	pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn);
+
 	/* Obtain current timeline */
 #if PG_VERSION_NUM >= 90600
 	current.tli = get_current_timeline(backup_conn);
 #else
 	current.tli = get_current_timeline_from_control(false);
 #endif
 
+	/* In PAGE mode or in ARCHIVE wal-mode wait for current segment */
+	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal)
+		/*
+		 * Do not wait start_lsn for stream backup.
+		 * Because WAL streaming will start after pg_start_backup() in stream
+		 * mode.
+		 */
+		wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false);
+
 	/*
 	 * In incremental backup mode ensure that already-validated
 	 * backup on current timeline exists and get its filelist.
@@ -252,29 +270,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 		}
 	}
 
-	/* Clear ptrack files for FULL and PAGE backup */
-	if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable)
-		pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num);
-
-	/* notify start of backup to PostgreSQL server */
-	time2iso(label, lengthof(label), current.start_time);
-	strncat(label, " with pg_probackup", lengthof(label) -
-			strlen(" with pg_probackup"));
-
-	/* Create connection to master server needed to call pg_start_backup */
-	if (current.from_replica && exclusive_backup)
-	{
-		master_conn = pgut_connect(instance_config.master_conn_opt.pghost,
-								   instance_config.master_conn_opt.pgport,
-								   instance_config.master_conn_opt.pgdatabase,
-								   instance_config.master_conn_opt.pguser);
-		pg_startbackup_conn = master_conn;
-	}
-	else
-		pg_startbackup_conn = backup_conn;
-
-	pg_start_backup(label, smooth_checkpoint, &current, nodeInfo, backup_conn, pg_startbackup_conn);
-
 	/* For incremental backup check that start_lsn is not from the past
 	 * Though it will not save us if PostgreSQL instance is actually
 	 * restored STREAM backup.
@@ -342,7 +337,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 	 * Get database_map (name to oid) for use in partial restore feature.
 	 * It's possible that we fail and database_map will be NULL.
 	 */
-	database_map = get_database_map(pg_startbackup_conn);
+	database_map = get_database_map(backup_conn);
 
 	/*
 	 * Append to backup list all files and directories
@@ -571,7 +566,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 	}
 
 	/* Notify end of backup */
-	pg_stop_backup(&current, pg_startbackup_conn, nodeInfo);
+	pg_stop_backup(&current, backup_conn, nodeInfo);
 
 	/* In case of backup from replica >= 9.6 we must fix minRecPoint,
 	 * First we must find pg_control in backup_files_list.
@@ -1101,19 +1096,15 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz)
  */
 static void
 pg_start_backup(const char *label, bool smooth, pgBackup *backup,
-				PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *pg_startbackup_conn)
+				PGNodeInfo *nodeInfo, PGconn *conn)
 {
 	PGresult   *res;
 	const char *params[2];
 	uint32		lsn_hi;
 	uint32		lsn_lo;
-	PGconn	   *conn;
 
 	params[0] = label;
 
-	/* For 9.5 replica we call pg_start_backup() on master */
-	conn = pg_startbackup_conn;
-
 	/* 2nd argument is 'fast'*/
 	params[1] = smooth ? "false" : "true";
 	if (!exclusive_backup)
@@ -1132,7 +1123,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 	 * is necessary to call pg_stop_backup() in backup_cleanup().
 	 */
 	backup_in_progress = true;
-	pgut_atexit_push(backup_stopbackup_callback, pg_startbackup_conn);
+	pgut_atexit_push(backup_stopbackup_callback, conn);
 
 	/* Extract timeline and LSN from results of pg_start_backup() */
 	XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo);
@@ -1152,15 +1143,6 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 		 * (because in 9.5 only superuser can switch WAL)
 		 */
 		pg_switch_wal(conn);
-
-	/* In PAGE mode or in ARCHIVE wal-mode wait for current segment */
-	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal)
-		/*
-		 * Do not wait start_lsn for stream backup.
-		 * Because WAL streaming will start after pg_start_backup() in stream
-		 * mode.
-		 */
-		wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false);
 }
 
 /*

tests/archive.py

@@ -765,10 +765,6 @@ def test_replica_archive(self):
 
         before = master.safe_psql("postgres", "SELECT * FROM t_heap")
 
-        master.safe_psql(
-            "postgres",
-            "CHECKPOINT")
-
         self.wait_until_replica_catch_with_master(master, replica)
 
         backup_id = self.backup_node(
@@ -864,10 +860,6 @@ def test_master_and_replica_parallel_archiving(self):
             "md5(repeat(i::text,10))::tsvector as tsvector "
             "from generate_series(0, 60000) i")
 
-        master.psql(
-            "postgres",
-            "CHECKPOINT")
-
         backup_id = self.backup_node(
             backup_dir, 'replica', replica,
             options=[
@@ -977,10 +969,6 @@ def test_basic_master_and_replica_concurrent_archiving(self):
 
         replica.promote()
 
-        replica.safe_psql(
-            'postgres',
-            'CHECKPOINT')
-
         master.pgbench_init(scale=10)
         replica.pgbench_init(scale=10)
 
@@ -1221,11 +1209,6 @@ def test_archive_catalog(self):
         # create timeline t2
         replica.promote()
 
-        # do checkpoint to increment timeline ID in pg_control
-        replica.safe_psql(
-            'postgres',
-            'CHECKPOINT')
-
         # FULL backup replica
         A1 = self.backup_node(
             backup_dir, 'replica', replica)
@@ -1959,7 +1942,8 @@ def test_archive_pg_receivexlog_partial_handling(self):
 
         replica.slow_start(replica=True)
 
-        node.safe_psql('postgres', 'CHECKPOINT')
+        # FULL
+        self.backup_node(backup_dir, 'replica', replica, options=['--stream'])
 
         if self.get_version(replica) < 100000:
             pg_receivexlog_path = self.get_bin_path('pg_receivexlog')
@@ -1981,14 +1965,18 @@ def test_archive_pg_receivexlog_partial_handling(self):
                 'Failed to start pg_receivexlog: {0}'.format(
                     pg_receivexlog.communicate()[1]))
 
+        replica.safe_psql(
+            'postgres',
+            'CHECKPOINT')
+
         node.safe_psql(
             "postgres",
             "create table t_heap as select i as id, md5(i::text) as text, "
             "md5(repeat(i::text,10))::tsvector as tsvector "
             "from generate_series(0,1000000) i")
 
-        # FULL
-        self.backup_node(backup_dir, 'replica', replica, options=['--stream'])
+        # PAGE
+        self.backup_node(backup_dir, 'replica', replica, backup_type='page')
 
         node.safe_psql(
             "postgres",
@@ -2027,6 +2015,7 @@ def test_archive_pg_receivexlog_partial_handling(self):
         pg_receivexlog.kill()
         self.del_test_dir(module_name, fname)
 
+    @unittest.skip("skip")
     def test_multi_timeline_recovery_prefetching(self):
         """"""
         fname = self.id().split('.')[3]

tests/backup.py

@@ -324,6 +324,7 @@ def test_page_detect_corruption(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True,
+            ptrack_enable=True,
             initdb_params=['--data-checksums'])
 
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
@@ -663,12 +664,10 @@ def test_backup_detect_invalid_block_header(self):
                     "\n Output: {0} \n CMD: {1}".format(
                         repr(self.output), self.cmd))
             except ProbackupException as e:
-                self.assertTrue(
-                    'WARNING:  page verification failed, '
-                    'calculated checksum' in e.message and
-                    'ERROR: query failed: ERROR:  '
-                    'invalid page in block 1 of relation' in e.message and
-                    'ERROR: Data files transferring failed' in e.message,
+                self.assertIn(
+                    'ERROR: Corruption detected in file "{0}", block 1: '
+                    'page header invalid, pd_lower'.format(heap_fullpath),
+                    e.message,
                     '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
                         repr(e.message), self.cmd))
 
@@ -806,12 +805,10 @@ def test_backup_detect_missing_permissions(self):
                     "\n Output: {0} \n CMD: {1}".format(
                         repr(self.output), self.cmd))
             except ProbackupException as e:
-                self.assertTrue(
-                    'WARNING:  page verification failed, '
-                    'calculated checksum' in e.message and
-                    'ERROR: query failed: ERROR:  '
-                    'invalid page in block 1 of relation' in e.message and
-                    'ERROR: Data files transferring failed' in e.message,
+                self.assertIn(
+                    'ERROR: Corruption detected in file "{0}", block 1: '
+                    'page header invalid, pd_lower'.format(heap_fullpath),
+                    e.message,
                     '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
                         repr(e.message), self.cmd))
 
@@ -1480,7 +1477,7 @@ def test_drop_rel_during_backup_ptrack(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True,
-            ptrack_enable=True,
+            ptrack_enable=self.ptrack,
             initdb_params=['--data-checksums'])
 
         self.init_pb(backup_dir)
@@ -2131,9 +2128,8 @@ def test_backup_with_least_privileges_role(self):
                         "TO backup".format(fname))
             else:
                 fnames = [
-                    'pg_catalog.pg_ptrack_get_pagemapset(pg_lsn)',
-                    'pg_catalog.pg_ptrack_control_lsn()',
-                    'pg_catalog.pg_ptrack_get_block(oid, oid, oid, bigint)'
+                    'pg_catalog.ptrack_get_pagemapset(pg_lsn)',
+                    'pg_catalog.ptrack_init_lsn()'
                 ]
 
                 for fname in fnames:

tests/ptrack.py

@@ -3946,9 +3946,9 @@ def test_ptrack_pg_resetxlog(self):
                     repr(self.output), self.cmd)
             )
         except ProbackupException as e:
-            self.assertIn(
-                'ERROR: LSN from ptrack_control 0/0 differs from Start LSN of previous backup',
-                e.message,
+            self.assertTrue(
+                'ERROR: LSN from ptrack_control ' in e.message and
+                'differs from Start LSN of previous backup' in e.message,
                 '\n Unexpected Error Message: {0}\n'
                 ' CMD: {1}'.format(repr(e.message), self.cmd))