Merge branch 'master' into release_2_4

Author: gsmolk

README.md

@@ -54,7 +54,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
 
 ## Current release
 
-[2.3.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.3)
+[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4)
 
 ## Documentation
 

src/archive.c

@@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 
 	/* get list of files from archive_status */
 	status_files = parray_new();
-	dir_list_file(status_files, archive_status_dir, false, false, false, false, 0, FIO_DB_HOST);
+	dir_list_file(status_files, archive_status_dir, false, false, false, false, true, 0, FIO_DB_HOST);
 	parray_qsort(status_files, pgFileCompareName);
 
 	for (i = 0; i < parray_num(status_files); i++)

src/backup.c

@@ -333,10 +333,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 	/* list files with the logical path. omit $PGDATA */
 	if (fio_is_remote(FIO_DB_HOST))
 		fio_list_dir(backup_files_list, instance_config.pgdata,
-					 true, true, false, backup_logs, 0);
+					 true, true, false, backup_logs, true, 0);
 	else
 		dir_list_file(backup_files_list, instance_config.pgdata,
-					  true, true, false, backup_logs, 0, FIO_LOCAL_HOST);
+					  true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST);
 
 	/*
 	 * Get database_map (name to oid) for use in partial restore feature.
@@ -356,10 +356,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 			 * 0 value is not external dir */
 			if (fio_is_remote(FIO_DB_HOST))
 				fio_list_dir(backup_files_list, parray_get(external_dirs, i),
-						  false, true, false, false, i+1);
+							 false, true, false, false, true, i+1);
 			else
 				dir_list_file(backup_files_list, parray_get(external_dirs, i),
-							  false, true, false, false, i+1, FIO_LOCAL_HOST);
+							  false, true, false, false, true, i+1, FIO_LOCAL_HOST);
 		}
 	}
 
@@ -615,7 +615,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
 		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, 0,
+		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, true, 0,
 					  FIO_BACKUP_HOST);
 
 		/* TODO: Drop streamed WAL segments greater than stop_lsn */

src/catalog.c

@@ -878,7 +878,7 @@ catalog_get_timelines(InstanceConfig *instance)
 
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, false, true, 0, FIO_BACKUP_HOST);
 	parray_qsort(xlog_files_list, pgFileCompareName);
 
 	timelineinfos = parray_new();

src/checkdb.c

@@ -208,7 +208,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 
 	/* list files with the logical path. omit $PGDATA */
 	dir_list_file(files_list, pgdata, true, true,
-				  false, false, 0, FIO_DB_HOST);
+				  false, false, true, 0, FIO_DB_HOST);
 
 	/*
 	 * Sort pathname ascending.

src/delete.c

@@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup)
 
 	/* list files to be deleted */
 	files = parray_new();
-	dir_list_file(files, backup->root_dir, false, false, true, false, 0, FIO_LOCAL_HOST);
+	dir_list_file(files, backup->root_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST);
 
 	/* delete leaf node first */
 	parray_qsort(files, pgFileCompareRelPathWithExternalDesc);

src/dir.c

@@ -124,8 +124,9 @@ typedef struct TablespaceCreatedList
 static int pgCompareString(const void *str1, const void *str2);
 
 static char dir_check_file(pgFile *file);
+
 static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
-								   bool exclude, bool follow_symlink,
+								   bool exclude, bool follow_symlink, bool skip_hidden,
 								   int external_dir_num, fio_location location);
 static void opt_path_map(ConfigOption *opt, const char *arg,
 						 TablespaceList *list, const char *type);
@@ -513,7 +514,8 @@ db_map_entry_free(void *entry)
  */
 void
 dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
-			  bool add_root, bool backup_logs, int external_dir_num, fio_location location)
+			  bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num,
+			  fio_location location)
 {
 	pgFile	   *file;
 
@@ -551,7 +553,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
 		parray_append(files, file);
 
 	dir_list_file_internal(files, file, root, exclude, follow_symlink,
-						   external_dir_num, location);
+						   skip_hidden, external_dir_num, location);
 
 	if (!add_root)
 		pgFileFree(file);
@@ -773,7 +775,7 @@ dir_check_file(pgFile *file)
  */
 static void
 dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
-					   bool exclude, bool follow_symlink,
+					   bool exclude, bool follow_symlink, bool skip_hidden,
 					   int external_dir_num, fio_location location)
 {
 	DIR			  *dir;
@@ -820,7 +822,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
 		}
 
 		/* skip hidden files and directories */
-		if (file->name[0] == '.')
+		if (skip_hidden && file->name[0] == '.')
 		{
 			elog(WARNING, "Skip hidden file: '%s'", child);
 			pgFileFree(file);
@@ -863,7 +865,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
 		 */
 		if (S_ISDIR(file->mode))
 			dir_list_file_internal(files, file, child, exclude, follow_symlink,
-								   external_dir_num, location);
+								   skip_hidden, external_dir_num, location);
 	}
 
 	if (errno && errno != ENOENT)

src/merge.c

@@ -1062,7 +1062,7 @@ remove_dir_with_files(const char *path)
 	int			i;
 	char 		full_path[MAXPGPATH];
 
-	dir_list_file(files, path, false, false, true, false, 0, FIO_LOCAL_HOST);
+	dir_list_file(files, path, false, false, true, false, false, 0, FIO_LOCAL_HOST);
 	parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
 	for (i = 0; i < parray_num(files); i++)
 	{

src/pg_probackup.c

@@ -561,9 +561,18 @@ main(int argc, char *argv[])
 		setMyLocation();
 	}
 
-	/* disable logging into file for archive-push and archive-get */
-	if (backup_subcmd == ARCHIVE_GET_CMD ||
-		backup_subcmd == ARCHIVE_PUSH_CMD)
+	/*
+	 * Disable logging into file for archive-push and archive-get.
+	 * Note, that we should NOT use fio_is_remote() here,
+	 * because it will launch ssh connection and we do not
+	 * want it, because it will kill archive-get prefetch
+	 * performance.
+	 *
+	 * TODO: make logging into file possible via ssh
+	 */
+	if (fio_is_remote_simple(FIO_BACKUP_HOST) &&
+		(backup_subcmd == ARCHIVE_GET_CMD ||
+		backup_subcmd == ARCHIVE_PUSH_CMD))
 	{
 		instance_config.logger.log_level_file = LOG_OFF;
 	}

src/pg_probackup.h

@@ -241,8 +241,8 @@ typedef enum ShowFormat
 #define BYTES_INVALID		(-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
 #define FILE_NOT_FOUND		(-2) /* file disappeared during backup */
 #define BLOCKNUM_INVALID	(-1)
-#define PROGRAM_VERSION	"2.3.3"
-#define AGENT_PROTOCOL_VERSION 20303
+#define PROGRAM_VERSION	"2.3.4"
+#define AGENT_PROTOCOL_VERSION 20304
 
 
 typedef struct ConnectionOptions
@@ -860,7 +860,7 @@ extern const char* deparse_compress_alg(int alg);
 /* in dir.c */
 extern void dir_list_file(parray *files, const char *root, bool exclude,
 						  bool follow_symlink, bool add_root, bool backup_logs,
-						  int external_dir_num, fio_location location);
+						  bool skip_hidden, int external_dir_num, fio_location location);
 
 extern void create_data_directories(parray *dest_files,
 										const char *data_dir,
@@ -1036,7 +1036,7 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL
 														pgFile *file, char **errormsg);
 
 extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
-						 bool add_root, bool backup_logs, int external_dir_num);
+						 bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num);
 
 extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
 
@@ -1052,6 +1052,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
 
 /* Check if specified location is local for current node */
 extern bool fio_is_remote(fio_location location);
+extern bool fio_is_remote_simple(fio_location location);
 
 extern void get_header_errormsg(Page page, char **errormsg);
 extern void get_checksum_errormsg(Page page, char **errormsg,

src/utils/file.c

@@ -44,6 +44,7 @@ typedef struct
 	bool add_root;
 	bool backup_logs;
 	bool exclusive_backup;
+	bool skip_hidden;
 	int  external_dir_num;
 } fio_list_dir_request;
 
@@ -175,6 +176,15 @@ bool fio_is_remote(fio_location location)
 	return is_remote;
 }
 
+/* Check if specified location is local for current node */
+bool fio_is_remote_simple(fio_location location)
+{
+	bool is_remote = MyLocation != FIO_LOCAL_HOST
+		&& location != FIO_LOCAL_HOST
+		&& location != MyLocation;
+	return is_remote;
+}
+
 /* Try to read specified amount of bytes unless error or EOF are encountered */
 static ssize_t fio_read_all(int fd, void* buf, size_t size)
 {
@@ -1536,7 +1546,7 @@ static void fio_send_pages_impl(int out, char* buf)
 			hdr.arg = OPEN_FAILED;
 			errormsg = pgut_malloc(ERRMSG_MAX_LEN);
 			/* Construct the error message */
-			snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open source file '%s': %s",
+			snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file \"%s\": %s",
 					 from_fullpath, strerror(errno));
 			hdr.size = strlen(errormsg) + 1;
 		}
@@ -2065,7 +2075,8 @@ static void fio_send_file_impl(int out, char const* path)
 
 /* Compile the array of files located on remote machine in directory root */
 void fio_list_dir(parray *files, const char *root, bool exclude,
-				  bool follow_symlink, bool add_root, bool backup_logs, int external_dir_num)
+				  bool follow_symlink, bool add_root, bool backup_logs,
+				  bool skip_hidden, int external_dir_num)
 {
 	fio_header hdr;
 	fio_list_dir_request req;
@@ -2078,6 +2089,7 @@ void fio_list_dir(parray *files, const char *root, bool exclude,
 	req.add_root = add_root;
 	req.backup_logs = backup_logs;
 	req.exclusive_backup = exclusive_backup;
+	req.skip_hidden = skip_hidden;
 	req.external_dir_num = external_dir_num;
 
 	hdr.cop = FIO_LIST_DIR;
@@ -2172,7 +2184,8 @@ static void fio_list_dir_impl(int out, char* buf)
 	exclusive_backup = req->exclusive_backup;
 
 	dir_list_file(file_files, req->path, req->exclude, req->follow_symlink,
-				req->add_root, req->backup_logs, req->external_dir_num, FIO_LOCAL_HOST);
+				  req->add_root, req->backup_logs, req->skip_hidden,
+				  req->external_dir_num, FIO_LOCAL_HOST);
 
 	/* send information about files to the main process */
 	for (i = 0; i < parray_num(file_files); i++)

tests/compatibility.py

@@ -958,3 +958,33 @@ def test_page_vacuum_truncate_compression(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_hidden_files(self):
+        """
+        old_version should be < 2.3.0
+        Create hidden file in pgdata, take backup
+        with old binary, then try to delete backup
+        with new binary
+        """
+        fname = self.id().split('.')[3]
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'],
+            pg_options={'autovacuum': 'off'})
+
+        self.init_pb(backup_dir, old_binary=True)
+        self.add_instance(backup_dir, 'node', node, old_binary=True)
+        node.slow_start()
+
+        open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close()
+
+        backup_id = self.backup_node(
+            backup_dir, 'node',node, old_binary=True, options=['--stream'])
+
+        self.delete_pb(backup_dir, 'node', backup_id)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)

tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 2.3.3
+pg_probackup 2.3.4