Remove name argument of make_bnode()

Author: Artur Zakirov

tests/backup_test.py

@@ -16,7 +16,7 @@ def tearDownClass(cls):
 
 	def test_backup_modes_1(self):
 		"""standart backup modes"""
-		node = self.make_bnode('backup_modes_', base_dir="tmp_dirs/backup/backup_modes_1")
+		node = self.make_bnode(base_dir="tmp_dirs/backup/backup_modes_1")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 
@@ -73,7 +73,7 @@ def test_backup_modes_1(self):
 
 	def test_smooth_checkpoint_2(self):
 		"""full backup with smooth checkpoint"""
-		node = self.make_bnode('smooth_checkpoint_2', base_dir="tmp_dirs/backup/smooth_checkpoint_2")
+		node = self.make_bnode(base_dir="tmp_dirs/backup/smooth_checkpoint_2")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 
@@ -86,7 +86,7 @@ def test_smooth_checkpoint_2(self):
 
 	def test_page_backup_without_full_3(self):
 		"""page-level backup without validated full backup"""
-		node = self.make_bnode('without_full_3', base_dir="tmp_dirs/backup/without_full_3")
+		node = self.make_bnode(base_dir="tmp_dirs/backup/without_full_3")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 
@@ -100,7 +100,6 @@ def test_page_backup_without_full_3(self):
 	def test_ptrack_threads_4(self):
 		"""ptrack multi thread backup mode"""
 		node = self.make_bnode(
-			'ptrack_threads_4',
 			base_dir="tmp_dirs/backup/ptrack_threads_4",
 			options={"ptrack_enable": "on"}
 		)
@@ -122,7 +121,6 @@ def test_ptrack_threads_4(self):
 	def test_ptrack_threads_stream_5(self):
 		"""ptrack multi thread backup mode and stream"""
 		node = self.make_bnode(
-			'ptrack_threads_stream_5',
 			base_dir="tmp_dirs/backup/ptrack_threads_stream_5",
 			options={
 				"ptrack_enable": "on",

tests/delete_test.py

@@ -17,7 +17,7 @@ def tearDownClass(cls):
 
 	def test_delete_full_backups_1(self):
 		"""delete full backups"""
-		node = self.make_bnode('delete_full_backups_1', base_dir="tmp_dirs/delete/delete_full_backups_1")
+		node = self.make_bnode(base_dir="tmp_dirs/delete/delete_full_backups_1")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init()
@@ -52,7 +52,7 @@ def test_delete_full_backups_1(self):
 
 	def test_delete_increment_2(self):
 		"""delete increment and all after him"""
-		node = self.make_bnode('delete_increment_2', base_dir="tmp_dirs/delete/delete_increment_2")
+		node = self.make_bnode(base_dir="tmp_dirs/delete/delete_increment_2")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 

tests/init_test.py

@@ -12,7 +12,7 @@ def __init__(self, *args, **kwargs):
 
 	def test_success_1(self):
 		"""Success normal init"""
-		node = self.make_bnode('test_success_1', base_dir="tmp_dirs/init/success_1")
+		node = self.make_bnode(base_dir="tmp_dirs/init/success_1")
 		self.assertEqual(self.init_pb(node), six.b(""))
 		self.assertEqual(
 			dir_files(self.backup_dir(node)),
@@ -21,7 +21,7 @@ def test_success_1(self):
 
 	def test_already_exist_2(self):
 		"""Failure with backup catalog already existed"""
-		node = self.make_bnode('test_already_exist_2', base_dir="tmp_dirs/init/already_exist_2")
+		node = self.make_bnode(base_dir="tmp_dirs/init/already_exist_2")
 		self.init_pb(node)
 		self.assertEqual(
 			self.init_pb(node),
@@ -30,7 +30,7 @@ def test_already_exist_2(self):
 
 	def test_abs_path_3(self):
 		"""failure with backup catalog should be given as absolute path"""
-		node = self.make_bnode('test_abs_path_3', base_dir="tmp_dirs/init/abs_path_3")
+		node = self.make_bnode(base_dir="tmp_dirs/init/abs_path_3")
 		self.assertEqual(
 			self.run_pb(["init", "-B", path.relpath("%s/backup" % node.base_dir, self.dir_path)]),
 			six.b("ERROR: -B, --backup-path must be an absolute path\n")

tests/option_test.py

@@ -38,7 +38,7 @@ def test_without_backup_path_3(self):
 		)
 
 	def test_options_4(self):
-		node = self.make_bnode('test_options_4', base_dir="tmp_dirs/option/option_common")
+		node = self.make_bnode(base_dir="tmp_dirs/option/option_common")
 		try:
 			node.stop()
 		except:

tests/pb_lib.py

@@ -91,18 +91,15 @@ def arcwal_dir(self, node):
 	def backup_dir(self, node):
 		return os.path.abspath("%s/backup" % node.base_dir)
 
-	def make_bnode(self, name, base_dir=None, options={}):
+	def make_bnode(self, base_dir=None, allows_streaming=False, options={}):
 		real_base_dir = path.join(self.dir_path, base_dir)
 		shutil.rmtree(real_base_dir, ignore_errors=True)
+
 		node = get_new_node('test', base_dir=real_base_dir)
-		node.init()
-		# node.init(initdb_params=[
-		# 	"-x", "0x123456789ABCDEF0",
-		# 	"-m", "0x23456789ABCDEF01",
-		# 	"-o", "0x3456789ABCDEF012"
-		# ])
-
-		node.append_conf("postgresql.conf", "wal_level = hot_standby")
+		node.init(allows_streaming=allows_streaming)
+
+		if not allows_streaming:
+			node.append_conf("postgresql.conf", "wal_level = hot_standby")
 		node.append_conf("postgresql.conf", "archive_mode = on")
 		node.append_conf(
 			"postgresql.conf",
@@ -114,6 +111,33 @@ def make_bnode(self, name, base_dir=None, options={}):
 
 		return node
 
+	def make_bnode_replica(self, root_node, base_dir=None, options={}):
+		real_base_dir = path.join(self.dir_path, base_dir)
+		shutil.rmtree(real_base_dir, ignore_errors=True)
+
+		root_node.backup("basebackup")
+
+		replica = get_new_node("replica", base_dir=real_base_dir)
+		# replica.init_from_backup(root_node, "data_replica", has_streaming=True)
+
+		# Move data from backup
+		backup_path = os.path.join(root_node.base_dir, "basebackup")
+		shutil.move(backup_path, replica.data_dir)
+		os.chmod(replica.data_dir, 0o0700)
+
+		# Change port in config file
+		replica.append_conf(
+			"postgresql.conf",
+			"port = {}".format(replica.port)
+		)
+		# Enable streaming
+		replica.enable_streaming(root_node)
+
+		for key, value in six.iteritems(options):
+			replica.append_conf("postgresql.conf", "%s = %s" % (key, value))
+
+		return replica
+
 	def run_pb(self, command):
 		try:
 			return subprocess.check_output(
@@ -145,9 +169,29 @@ def backup_pb(self, node, backup_type="full", options=[]):
 		if backup_type:
 			cmd_list += ["-b", backup_type]
 
-		# print(cmd_list)
 		return self.run_pb(cmd_list + options)
 
+	def backup_pb_proc(self, node, backup_dir, backup_type="full",
+		stdout=None, stderr=None, options=[]):
+		cmd_list = [
+			self.probackup_path,
+			"backup",
+			"-D", node.data_dir,
+			"-B", backup_dir,
+			"-p", "%i" % (node.port),
+			"-d", "postgres"
+		]
+		if backup_type:
+			cmd_list += ["-b", backup_type]
+
+		proc = subprocess.Popen(
+			cmd_list + options,
+			stdout=stdout,
+			stderr=stderr
+		)
+
+		return proc
+
 	def restore_pb(self, node, id=None, options=[]):
 		cmd_list = [
 			"-D", node.data_dir,

tests/restore_test.py

@@ -19,7 +19,7 @@ def tearDownClass(cls):
 
 	def test_restore_to_latest_1(self):
 		"""recovery to latest from full backup"""
-		node = self.make_bnode('restore_to_latest_1', base_dir="tmp_dirs/restore/restore_to_latest_1")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_latest_1")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -50,7 +50,7 @@ def test_restore_to_latest_1(self):
 
 	def test_restore_to_latest_2(self):
 		"""recovery to latest from full + page backups"""
-		node = self.make_bnode('restore_to_latest_2', base_dir="tmp_dirs/restore/restore_to_latest_2")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_latest_2")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -82,7 +82,7 @@ def test_restore_to_latest_2(self):
 
 	def test_restore_to_timeline_3(self):
 		"""recovery to target timeline"""
-		node = self.make_bnode('restore_to_timeline_3', base_dir="tmp_dirs/restore/restore_to_timeline_3")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_timeline_3")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -127,7 +127,7 @@ def test_restore_to_timeline_3(self):
 
 	def test_restore_to_time_4(self):
 		"""recovery to target timeline"""
-		node = self.make_bnode('restore_to_time_4', base_dir="tmp_dirs/restore/restore_to_time_4")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_time_4")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -158,7 +158,7 @@ def test_restore_to_time_4(self):
 
 	def test_restore_to_xid_5(self):
 		"""recovery to target xid"""
-		node = self.make_bnode('restore_to_xid_5', base_dir="tmp_dirs/restore/restore_to_xid_5")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_xid_5")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -204,7 +204,7 @@ def test_restore_to_xid_5(self):
 
 	def test_restore_full_ptrack_6(self):
 		"""recovery to latest from full + ptrack backups"""
-		node = self.make_bnode('restore_full_ptrack_6', base_dir="tmp_dirs/restore/full_ptrack_6")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/full_ptrack_6")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -244,7 +244,7 @@ def test_restore_full_ptrack_6(self):
 
 	def test_restore_full_ptrack_ptrack_7(self):
 		"""recovery to latest from full + ptrack + ptrack backups"""
-		node = self.make_bnode('restore_full_ptrack_ptrack_7', base_dir="tmp_dirs/restore/full_ptrack_ptrack_7")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/full_ptrack_ptrack_7")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -291,7 +291,7 @@ def test_restore_full_ptrack_ptrack_7(self):
 
 	def test_restore_full_ptrack_stream_8(self):
 		"""recovery in stream mode to latest from full + ptrack backups"""
-		node = self.make_bnode('restore_full_ptrack_stream_8', base_dir="tmp_dirs/restore/full_ptrack_stream_8")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/full_ptrack_stream_8")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -334,7 +334,7 @@ def test_restore_full_ptrack_stream_8(self):
 
 	def test_restore_full_ptrack_under_load_9(self):
 		"""recovery to latest from full + page backups with loads when ptrack backup do"""
-		node = self.make_bnode('restore_full_ptrack_under_load_9', base_dir="tmp_dirs/restore/full_ptrack_under_load_9")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/full_ptrack_under_load_9")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		wal_segment_size = self.guc_wal_segment_size(node)
@@ -391,7 +391,7 @@ def test_restore_full_ptrack_under_load_9(self):
 
 	def test_restore_full_under_load_ptrack_10(self):
 		"""recovery to latest from full + page backups with loads when full backup do"""
-		node = self.make_bnode('restore_full_under_load_ptrack_10', base_dir="tmp_dirs/restore/full_under_load_ptrack_10")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/full_under_load_ptrack_10")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		wal_segment_size = self.guc_wal_segment_size(node)
@@ -448,7 +448,7 @@ def test_restore_full_under_load_ptrack_10(self):
 
 	def test_restore_to_xid_inclusive_11(self):
 		"""recovery with target inclusive false"""
-		node = self.make_bnode('restore_to_xid_inclusive_11', base_dir="tmp_dirs/restore/restore_to_xid_inclusive_11")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_to_xid_inclusive_11")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)
@@ -499,8 +499,7 @@ def test_restore_to_xid_inclusive_11(self):
 
 	def test_restore_with_tablespace_mapping_12(self):
 		"""recovery using tablespace-mapping option"""
-		node = self.make_bnode('restore_with_tablespace_mapping_12',
-			base_dir="tmp_dirs/restore/restore_with_tablespace_mapping_12")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_with_tablespace_mapping_12")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 
@@ -564,8 +563,7 @@ def test_restore_with_tablespace_mapping_12(self):
 
 	def test_restore_with_tablespace_mapping_13(self):
 		"""recovery using tablespace-mapping option and page backup"""
-		node = self.make_bnode('restore_with_tablespace_mapping_13',
-			base_dir="tmp_dirs/restore/restore_with_tablespace_mapping_13")
+		node = self.make_bnode(base_dir="tmp_dirs/restore/restore_with_tablespace_mapping_13")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 

tests/retention_test.py

@@ -17,8 +17,7 @@ def tearDownClass(cls):
 
 	def test_retention_redundancy_1(self):
 		"""purge backups using redundancy-based retention policy"""
-		node = self.make_bnode('retention_redundancy_1',
-			base_dir="tmp_dirs/retention/retention_redundancy_1")
+		node = self.make_bnode(base_dir="tmp_dirs/retention/retention_redundancy_1")
 		node.start()
 
 		self.init_pb(node)
@@ -56,8 +55,7 @@ def test_retention_redundancy_1(self):
 
 	def test_retention_window_2(self):
 		"""purge backups using window-based retention policy"""
-		node = self.make_bnode('retention_window_2',
-			base_dir="tmp_dirs/retention/retention_window_2")
+		node = self.make_bnode(base_dir="tmp_dirs/retention/retention_window_2")
 		node.start()
 
 		self.init_pb(node)

tests/show_test.py

@@ -17,7 +17,7 @@ def tearDownClass(cls):
 
 	def test_ok_1(self):
 		"""Status DONE and OK"""
-		node = self.make_bnode('done_ok', base_dir="tmp_dirs/show/ok_1")
+		node = self.make_bnode(base_dir="tmp_dirs/show/ok_1")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 
@@ -31,7 +31,7 @@ def test_ok_1(self):
 
 	def test_corrupt_2(self):
 		"""Status DONE and OK"""
-		node = self.make_bnode('corrupt', base_dir="tmp_dirs/show/corrupt_2")
+		node = self.make_bnode(base_dir="tmp_dirs/show/corrupt_2")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 

tests/validate_test.py

@@ -21,7 +21,7 @@ def tearDownClass(cls):
 
 	def test_validate_wal_1(self):
 		"""recovery to latest from full backup"""
-		node = self.make_bnode('test_validate_wal_1', base_dir="tmp_dirs/validate/wal_1")
+		node = self.make_bnode(base_dir="tmp_dirs/validate/wal_1")
 		node.start()
 		self.assertEqual(self.init_pb(node), six.b(""))
 		node.pgbench_init(scale=2)