diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..c3ad89568 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,93 @@ +name: Build Probackup + +on: + push: + branches: + - "**" + # Runs triggered by pull requests are disabled to prevent executing potentially unsafe code from public pull requests + # pull_request: + # branches: + # - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + + build-win2019: + + runs-on: + - windows-2019 + + env: + zlib_dir: C:\dep\zlib + + steps: + + - uses: actions/checkout@v2 + + - name: Install pacman packages + run: | + $env:PATH += ";C:\msys64\usr\bin" + pacman -S --noconfirm --needed bison flex + + - name: Make zlib + run: | + git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib.git + cd zlib + cmake -DCMAKE_INSTALL_PREFIX:PATH=C:\dep\zlib -G "Visual Studio 16 2019" . + cmake --build . --config Release --target ALL_BUILD + cmake --build . --config Release --target INSTALL + copy C:\dep\zlib\lib\zlibstatic.lib C:\dep\zlib\lib\zdll.lib + copy C:\dep\zlib\bin\zlib.dll C:\dep\zlib\lib + + - name: Get Postgres sources + run: git clone -b REL_14_STABLE https://github.com/postgres/postgres.git + + # Copy ptrack to contrib to build the ptrack extension + # Convert line breaks in the patch file to LF otherwise the patch doesn't apply + - name: Get Ptrack sources + run: | + git clone -b master --depth 1 https://github.com/postgrespro/ptrack.git + Copy-Item -Path ptrack -Destination postgres\contrib -Recurse + (Get-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Raw).Replace("`r`n","`n") | Set-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Force -NoNewline + cd postgres + git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff + + - name: Build Postgres + run: | + $env:PATH += ";C:\msys64\usr\bin" + cd postgres\src\tools\msvc + (Get-Content config_default.pl) -Replace "zlib *=>(.*?)(?=,? *#)", "zlib => '${{ env.zlib_dir }}'" | Set-Content config.pl + cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && .\build.bat" + + - name: Build Probackup + run: cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && perl .\gen_probackup_project.pl `"${{ github.workspace }}`"\postgres" + + - name: Install Postgres + run: | + cd postgres + src\tools\msvc\install.bat postgres_install + + - name: Install Testgres + run: | + git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git + pip3 install psycopg2 ./testgres + + # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder + - name: Test Probackup + run: | + icacls.exe "${{ github.workspace }}" /grant "${env:USERNAME}:(OI)(CI)F" + $env:PATH += ";${{ github.workspace }}\postgres\postgres_install\lib;${{ env.zlib_dir }}\lib" + $Env:LC_MESSAGES = "English" + $Env:PG_CONFIG = "${{ github.workspace }}\postgres\postgres_install\bin\pg_config.exe" + $Env:PGPROBACKUPBIN = "${{ github.workspace }}\postgres\Release\pg_probackup\pg_probackup.exe" + $Env:PG_PROBACKUP_PTRACK = "ON" + If (!$Env:MODE -Or $Env:MODE -Eq "basic") { + $Env:PG_PROBACKUP_TEST_BASIC = "ON" + python -m unittest -v tests + python -m unittest -v tests.init_test + } else { + python -m unittest -v tests.$Env:MODE + } + diff --git a/.gitignore b/.gitignore index 0258e4e2e..97d323ceb 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ # Binaries /pg_probackup +# Generated translated file +/po/ru.mo + # Generated by test suite /regression.diffs /regression.out @@ -34,9 +37,6 @@ # Extra files /src/pg_crc.c -/src/datapagemap.c -/src/datapagemap.h -/src/logging.h /src/receivelog.c /src/receivelog.h /src/streamutil.c @@ -53,9 +53,15 @@ /docker-compose.yml /Dockerfile /Dockerfile.in -/run_tests.sh /make_dockerfile.sh /backup_restore.sh +# Packaging +/build +/packaging/pkg/tarballs/pgpro.tar.bz2 +/packaging/repo/pg_probackup +/packaging/repo/pg_probackup-forks + # Misc .python-version +.vscode diff --git a/.travis.yml b/.travis.yml index fc7ecc059..074ae3d02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,60 @@ os: linux -dist: bionic +dist: jammy language: c -services: - - docker +cache: ccache + +addons: + apt: + packages: + - sudo + - libc-dev + - bison + - flex + - libreadline-dev + - zlib1g-dev + - libzstd-dev + - libssl-dev + - perl + - libperl-dev + - libdbi-perl + - cpanminus + - locales + - python3 + - python3-dev + - python3-pip + - libicu-dev + - libgss-dev + - libkrb5-dev + - libxml2-dev + - libxslt1-dev + - libldap2-dev + - tcl-dev + - diffutils + - gdb + - gettext + - lcov + - openssh-client + - openssh-server + - libipc-run-perl + - libtime-hires-perl + - libtimedate-perl + - libdbd-pg-perl before_install: - - cp travis/* . + - sudo travis/before-install.sh install: - - ./make_dockerfile.sh - - docker-compose build + - travis/install.sh + +before_script: + - sudo travis/before-script.sh + - travis/before-script-user.sh script: - - docker-compose run tests - # - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - # - docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh + - travis/script.sh notifications: email: @@ -26,22 +63,40 @@ notifications: # Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON env: - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=archive - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=backup - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=compression - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=delta - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=locking - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=merge - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=page - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=replica - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=retention - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE MODE=restore - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE + - PG_VERSION=16 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE + - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE + - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE + - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE + - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup + - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup_stream +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=merge +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=option +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=page +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=ptrack +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=retention +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=restore +# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=time_consuming jobs: allow_failures: - - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) + - if: env(PG_BRANCH) = master + - if: env(PG_BRANCH) = REL9_5_STABLE +# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) + +# Only run CI for master branch commits to limit our travis usage +#branches: +# only: +# - master + diff --git a/Documentation.md b/Documentation.md deleted file mode 100644 index 943ea0cfd..000000000 --- a/Documentation.md +++ /dev/null @@ -1,2096 +0,0 @@ -# pg_probackup - -pg_probackup is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. pg_probackup supports PostgreSQL 9.5 or higher. - -Current version - 2.2.5 - -1. [Synopsis](#synopsis) -2. [Versioning](#versioning) -3. [Overview](#overview) - * [Limitations](#limitations) - -4. [Installation and Setup](#installation-and-setup) - * [Initializing the Backup Catalog](#initializing-the-backup-catalog) - * [Adding a New Backup Instance](#adding-a-new-backup-instance) - * [Configuring the Database Cluster](#configuring-the-database-cluster) - * [Setting up STREAM Backups](#setting-up-stream-backups) - * [Setting up Continuous WAL Archiving](#setting-up-continuous-wal-archiving) - * [Setting up Backup from Standby](#setting-up-backup-from-standby) - * [Setting up Cluster Verification](#setting-up-cluster-verification) - * [Setting up Partial Restore](#setting-up-partial-restore) - * [Configuring the Remote Mode](#configuring-the-remote-mode) - * [Setting up PTRACK Backups](#setting-up-ptrack-backups) - -5. [Usage](#usage) - * [Creating a Backup](#creating-a-backup) - * [ARCHIVE WAL mode](#archive-mode) - * [STREAM WAL mode](#stream-mode) - * [Page validation](#page-validation) - * [External directories](#external-directories) - * [Verifying a Cluster](#verifying-a-cluster) - * [Validating a Backup](#validating-a-backup) - * [Restoring a Cluster](#restoring-a-cluster) - * [Partial Restore](#partial-restore) - * [Performing Point-in-Time (PITR) Recovery](#performing-point-in-time-pitr-recovery) - * [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode) - * [Running pg_probackup on Parallel Threads](#running-pg_probackup-on-parallel-threads) - * [Configuring pg_probackup](#configuring-pg_probackup) - * [Managing the Backup Catalog](#managing-the-backup-catalog) - * [Viewing Backup Information](#viewing-backup-information) - * [Viewing WAL Archive Information](#viewing-wal-archive-information) - * [Configuring Retention Policy](#configuring-retention-policy) - * [Backup Retention Policy](#backup-retention-policy) - * [Backup Pinning](#backup-pinning) - * [WAL Archive Retention Policy](#wal-archive-retention-policy) - * [Merging Backups](#merging-backups) - * [Deleting Backups](#deleting-backups) - -6. [Command-Line Reference](#command-line-reference) - * [Commands](#commands) - * [version](#version) - * [help](#help) - * [init](#init) - * [add-instance](#add-instance) - * [del-instance](#del-instance) - * [set-config](#set-config) - * [set-backup](#set-backup) - * [show-config](#show-config) - * [show](#show) - * [backup](#backup) - * [restore](#restore) - * [checkdb](#checkdb) - * [validate](#validate) - * [merge](#merge) - * [delete](#delete) - * [archive-push](#archive-push) - * [archive-get](#archive-get) - * [Options](#options) - * [Common Options](#common-options) - * [Recovery Target Options](#recovery-target-options) - * [Retention Options](#retention-options) - * [Pinning Options](#pinning-options) - * [Logging Options](#logging-options) - * [Connection Options](#connection-options) - * [Compression Options](#compression-options) - * [Archiving Options](#archiving-options) - * [Remote Mode Options](#remote-mode-options) - * [Remote WAL Archive Options](#remote-wal-archive-options) - * [Partial Restore Options](#partial-restore-options) - * [Replica Options](#replica-options) - -7. [Howto](#howto) - * [Minimal setup](#minimal-setup) -8. [Authors](#authors) -9. [Credits](#credits) - - -## Synopsis - -`pg_probackup version` - -`pg_probackup help [command]` - -`pg_probackup init -B backup_dir` - -`pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name` - -`pg_probackup del-instance -B backup_dir --instance instance_name` - -`pg_probackup set-config -B backup_dir --instance instance_name [option...]` - -`pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id [option...]` - -`pg_probackup show-config -B backup_dir --instance instance_name [--format=format]` - -`pg_probackup show -B backup_dir [option...]` - -`pg_probackup backup -B backup_dir --instance instance_name -b backup_mode [option...]` - -`pg_probackup restore -B backup_dir --instance instance_name [option...]` - -`pg_probackup checkdb -B backup_dir --instance instance_name [-D data_dir] [option...]` - -`pg_probackup validate -B backup_dir [option...]` - -`pg_probackup merge -B backup_dir --instance instance_name -i backup_id [option...]` - -`pg_probackup delete -B backup_dir --instance instance_name { -i backup_id | --delete-wal | --delete-expired | --merge-expired } [option...]` - -`pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` - -`pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [option...]` - - -## Versioning - -pg_probackup is following the [semantic](https://semver.org/) versioning. - -## Overview - -As compared to other backup solutions, pg_probackup offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: - -- Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes you can plan the backup strategy in accordance with your data flow -- Validation: automatic data consistency checks and on-demand backup validation without actual data recovery -- Verification: on-demand verification of PostgreSQL instance via dedicated command `checkdb` -- Retention: managing WAL archive and backups in accordance with retention policies - Time and/or Redundancy based, with two retention methods: `delete expired` and `merge expired`. Additionally you can design you own retention policy by setting 'time to live' for backups -- Parallelization: running backup, restore, merge, delete, verificaton and validation processes on multiple parallel threads -- Compression: storing backup data in a compressed state to save disk space -- Deduplication: saving disk space by not copying the not changed non-data files ('_vm', '_fsm', etc) -- Remote operations: backup PostgreSQL instance located on remote machine or restore backup on it -- Backup from replica: avoid extra load on the master server by taking backups from a standby -- External directories: add to backup content of directories located outside of the PostgreSQL data directory (PGDATA), such as scripts, configs, logs and pg_dump files -- Backup Catalog: get list of backups and corresponding meta information in `plain` or `json` formats -- Archive Catalog: get list of all WAL timelines and corresponding meta information in `plain` or `json` formats -- Partial Restore: restore only the specified databases or skip the specified databases. - -To manage backup data, pg_probackup creates a `backup catalog`. This is a directory that stores all backup files with additional meta information, as well as WAL archives required for point-in-time recovery. You can store backups for different instances in separate subdirectories of a single backup catalog. - -Using pg_probackup, you can take full or incremental [backups](#creating-a-backup): - -- FULL backups contain all the data files required to restore the database cluster. -- Incremental backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup and restore operations. pg_probackup supports the following modes of incremental backups: - - DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that has changed since the previous backup. Note that this mode can impose read-only I/O pressure equal to a full backup. - - PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space. You have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) to make PAGE backups. - - PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special PTRACK bitmap for this relation. As one page requires just one bit in the PTRACK fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly. - -pg_probackup can take only physical online backups, and online backups require WAL for consistent recovery. So regardless of the chosen backup mode (FULL, PAGE or DELTA), any backup taken with pg_probackup must use one of the following `WAL delivery modes`: - -- [ARCHIVE](#archive-mode). Such backups rely on [continuous archiving](#setting-up-continuous-wal-archiving) to ensure consistent recovery. This is the default WAL delivery mode. -- [STREAM](#stream-mode). Such backups include all the files required to restore the cluster to a consistent state at the time the backup was taken. Regardless of [continuous archiving](#setting-up-continuous-wal-archiving) been set up or not, the WAL segments required for consistent recovery are streamed (hence STREAM) via replication protocol during backup and included into the backup files. Because of that backups of this WAL mode are called `autonomous` or `standalone`. - -### Limitations - -pg_probackup currently has the following limitations: - -- Only PostgreSQL of versions 9.5 and newer are supported. -- Currently remode mode of operations is not supported on Windows systems. -- On Unix systems backup of PostgreSQL verions =< 10 is possible only by the same OS user PostgreSQL server is running by. For example, if PostgreSQL server is running by user *postgres*, then backup must be run by user *postgres*. If backup is running in [remote mode](#using-pg_probackup-in-the-remote-mode) using `ssh`, then this limitation apply differently: value for `--remote-user` option should be *postgres*. -- During backup of PostgreSQL 9.5 functions `pg_create_restore_point(text)` and `pg_switch_xlog()` will be executed only if backup role is superuser. Because of that backup of a cluster with low amount of WAL traffic with non-superuser role may take more time than backup of the same cluster with superuser role. -- The PostgreSQL server from which the backup was taken and the restored server must be compatible by the [block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE) and [wal_block_size](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE) parameters and have the same major release number. Also depending on cluster configuration PostgreSQL itself may apply additional restrictions such as CPU architecture platform and libc/libicu versions. -- Incremental chain can span only within one timeline. So if you have backup incremental chain taken from replica and it gets promoted, you would be forced to take another FULL backup. - -## Installation and Setup - -Once you have pg_probackup installed, complete the following setup: - -- Initialize the backup catalog. -- Add a new backup instance to the backup catalog. -- Configure the database cluster to enable pg_probackup backups. -- Optionally, configure SSH for running pg_probackup operations in remote mode. - -### Initializing the Backup Catalog - -pg_probackup stores all WAL and backup files in the corresponding subdirectories of the backup catalog. - -To initialize the backup catalog, run the following command: - - pg_probackup init -B backup_dir - -Where *backup_dir* is the path to backup catalog. If the *backup_dir* already exists, it must be empty. Otherwise, pg_probackup returns an error. - -The user launching pg_probackup must have full access to *backup_dir* directory. - -pg_probackup creates the backup_dir backup catalog, with the following subdirectories: - -- wal/ — directory for WAL files. -- backups/ — directory for backup files. - -Once the backup catalog is initialized, you can add a new backup instance. - -### Adding a New Backup Instance - -pg_probackup can store backups for multiple database clusters in a single backup catalog. To set up the required subdirectories, you must add a backup instance to the backup catalog for each database cluster you are going to back up. - -To add a new backup instance, run the following command: - - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] - -Where: - -- *data_dir* is the data directory of the cluster you are going to back up. To set up and use pg_probackup, write access to this directory is required. -- *instance_name* is the name of the subdirectories that will store WAL and backup files for this cluster. -- The optional parameters [remote_options](#remote-mode-options) should be used if *data_dir* is located on remote machine. - -pg_probackup creates the *instance_name* subdirectories under the 'backups/' and 'wal/' directories of the backup catalog. The 'backups/*instance_name*' directory contains the 'pg_probackup.conf' configuration file that controls pg_probackup settings for this backup instance. If you run this command with the [remote_options](#remote-mode-options), used parameters will be added to pg_probackup.conf. - -For details on how to fine-tune pg_probackup configuration, see the section [Configuring pg_probackup](#configuring-pg_probackup). - -The user launching pg_probackup must have full access to *backup_dir* directory and at least read-only access to *data_dir* directory. If you specify the path to the backup catalog in the `BACKUP_PATH` environment variable, you can omit the corresponding option when running pg_probackup commands. - ->NOTE: For PostgreSQL >= 11 it is recommended to use [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature, so backup can be done by any OS user in the same group as the cluster owner. In this case the user should have read permissions on the cluster directory. - -### Configuring the Database Cluster - -Although pg_probackup can be used by a superuser, it is recommended to create a separate role with the minimum permissions required for the chosen backup strategy. In these configuration instructions, the *backup* role is used as an example. - -To perform [backup](#backup), the following permissions for role *backup* are required only in database **used for connection** to PostgreSQL server: - -For PostgreSQL 9.5: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -COMMIT; -``` - -For PostgreSQL 9.6: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; -``` - -For PostgreSQL >= 10: -``` -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; -``` - -In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow connection to database cluster on behalf of the *backup* role. - -Since pg_probackup needs to read cluster files directly, pg_probackup must be started by (in case of remote backup - connected to) OS user that has read access to all files and directories inside the data directory (PGDATA) you are going to back up. - -Depending on whether you are plan to take [autonomous](#stream-mode) and/or [archive](#archive-mode) backups, PostgreSQL cluster configuration will differ, as specified in the sections below. To back up the database cluster from a standby server, run pg_probackup in remote mode or create PTRACK backups, additional setup is required. - -For details, see the sections [Setting up STREAM Backups](#setting-up-stream-backups), [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving), [Setting up Backup from Standby](#setting-up-backup-from-standby), [Configuring the Remote Mode](#configuring-the-remote-mode), [Setting up Partial Restore](#setting-up-partial-restore) and [Setting up PTRACK Backups](#setting-up-ptrack-backups). - -### Setting up STREAM Backups - -To set up the cluster for [STREAM](#stream-mode) backups, complete the following steps: - -- Grant the REPLICATION privilege to the backup role: - - ALTER ROLE backup WITH REPLICATION; - -- In the [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file, allow replication on behalf of the *backup* role. -- Make sure the parameter [max_wal_senders](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) is set high enough to leave at least one session available for the backup process. -- Set the parameter [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) to be higher than `minimal`. - -If you are planning to take PAGE backups in STREAM mode or perform PITR with STREAM backups, you still have to configure WAL archiving as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). - -Once these steps are complete, you can start taking FULL, PAGE, DELTA and PTRACK backups with [STREAM](#stream-mode) WAL mode. - -### Setting up continuous WAL archiving - -Making backups in PAGE backup mode, performing [PITR](#performing-point-in-time-pitr-recovery) and making backups with [ARCHIVE](#archive-mode) WAL delivery mode require [continuous WAL archiving](https://www.postgresql.org/docs/current/continuous-archiving.html) to be enabled. To set up continuous archiving in the cluster, complete the following steps: - -- Make sure the [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) parameter is higher than `minimal`. -- If you are configuring archiving on master, [archive_mode](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE) must be set to `on` or `always`. To perform archiving on standby, set this parameter to `always`. -- Set the [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) parameter, as follows: - - archive_command = 'pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f [remote_options]' - -Where *backup_dir* and *instance_name* refer to the already initialized backup catalog instance for this database cluster and optional parameters [remote_options](#remote-mode-options) should be used to archive WAL to the remote host. For details about all possible `archive-push` parameters, see the section [archive-push](#archive-push). - -Once these steps are complete, you can start making backups with [ARCHIVE](#archive-mode) WAL-mode, backups in PAGE backup mode and perform [PITR](#performing-point-in-time-pitr-recovery). - -Current state of WAL Archive can be obtained via [show](#show) command. For details, see the sections [Viewing WAL Archive information](#viewing-wal-archive-information). - -If you are planning to make PAGE backups and/or backups with [ARCHIVE](#archive-mode) WAL mode from a standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up, consider setting [archive_timeout](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-TIMEOUT) PostgreSQL parameter **on master**. It is advisable to set the value of this setting slightly lower than pg_probackup parameter `--archive-timeout` (default 5 min), so there should be enough time for rotated segment to be streamed to replica and send to archive before backup is aborted because of `--archive-timeout`. - ->NOTE: using pg_probackup command [archive-push](#archive-push) for continuous archiving is optional. You can use any other tool you like as long as it delivers WAL segments into '*backup_dir*/wal/*instance_name*' directory. If compression is used, it should be `gzip`, and '.gz' suffix in filename is mandatory. - ->NOTE: Instead of `archive_mode`+`archive_command` method you may opt to use the utility [pg_receivewal](https://www.postgresql.org/docs/current/app-pgreceivewal.html). In this case pg_receivewal `-D directory` option should point to '*backup_dir*/wal/*instance_name*' directory. WAL compression that could be done by pg_receivewal is supported by pg_probackup. `Zero Data Loss` archive strategy can be achieved only by using pg_receivewal. - -### Setting up Backup from Standby - -For PostgreSQL 9.6 or higher, pg_probackup can take backups from a standby server. This requires the following additional setup: - -- On the standby server, set the parameter [hot_standby](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-HOT-STANDBY) to `on`. -- On the master server, set the parameter [full_page_writes](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-FULL-PAGE-WRITES) to `on`. -- To perform autonomous backups on standby, complete all steps in section [Setting up STREAM Backups](#setting-up-stream-backups) -- To perform archive backups on standby, complete all steps in section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving) - -Once these steps are complete, you can start taking FULL, PAGE, DELTA or PTRACK backups with appropriate WAL delivery mode: ARCHIVE or STREAM, from the standby server. - -Backup from the standby server has the following limitations: - -- If the standby is promoted to the master during backup, the backup fails. -- All WAL records required for the backup must contain sufficient full-page writes. This requires you to enable `full_page_writes` on the master, and not to use a tools like pg_compresslog as [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) to remove full-page writes from WAL files. - -### Setting up Cluster Verification - -Logical verification of database cluster requires the following additional setup. Role *backup* is used as an example: - -- Install extension [amcheck](https://www.postgresql.org/docs/current/amcheck.html) or [amcheck_next](https://github.com/petergeoghegan/amcheck) **in every database** of the cluster: - - CREATE EXTENSION amcheck; - -- To perform logical verification the following permissions are required **in every database** of the cluster: - -``` -GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; -GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid) TO backup; -GRANT EXECUTE ON FUNCTION bt_index_check(oid, bool) TO backup; -``` - -### Setting up Partial Restore - -If you are plalling to use partial restore, complete the following additional step: - -- Grant the read-only acces to 'pg_catalog.pg_database' to the *backup* role only in database **used for connection** to PostgreSQL server: - - GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; - -### Configuring the Remote Mode - -pg_probackup supports the remote mode that allows to perform backup, restore and WAL archiving operations remotely. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to backup and/or to restore is located on a remote system. Currently the only supported remote protocol is SSH. - -#### Setup SSH - -If you are going to use pg_probackup in remote mode via ssh, complete the following steps: - -- Install pg_probackup on both systems: `backup_host` and `db_host`. -- For communication between the hosts setup the passwordless SSH connection between *backup* user on `backup_host` and *postgres* user on `db_host`: - - [backup@backup_host] ssh-copy-id postgres@db_host - -- If you are planning to rely on [continuous WAL archiving](#setting-up-continuous-wal-archiving), then setup passwordless SSH connection between *postgres* user on `db_host` and *backup* user on `backup_host`: - - [postgres@db_host] ssh-copy-id backup@backup_host - -Where: - -- *backup_host* is the system with *backup catalog*. -- *db_host* is the system with PostgreSQL cluster. -- *backup* is the OS user on *backup_host* used to run pg_probackup. -- *postgres* is the OS user on *db_host* used to run PostgreSQL cluster. Note, that for PostgreSQL versions >= 11, a more secure approach can used thanks to [allow-group-access](https://www.postgresql.org/docs/11/app-initdb.html#APP-INITDB-ALLOW-GROUP-ACCESS) feature. - -pg_probackup in remote mode via `ssh` works as follows: - -- only the following commands can be launched in remote mode: [add-instance](#add-instance), [backup](#backup), [restore](#restore), [archive-push](#archive-push), [archive-get](#archive-get). -- when started in remote mode the main pg_probackup process on local system connects via ssh to remote system and launches there number of agent proccesses equal to specified value of option `-j/--threads`. -- the main pg_probackup process use remote agents to access remote files and transfer data between local and remote systems. -- remote agents are smart and capable of handling some logic on their own to minimize the network traffic and number of round-trips between hosts. -- usually the main proccess is started on *backup_host* and connects to *db_host*, but in case of `archive-push` and `archive-get` commands the main process is started on *db_host* and connects to *backup_host*. -- after completition of data transfer the remote agents are terminated and ssh connections are closed. -- if an error condition is encountered by a remote agent, then all agents are terminated and error details are reported by the main pg_probackup process, which exits with error. -- compression is always done on *db_host*. -- decompression is always done on *backup_host*. - ->NOTE: You can improse [additional restrictions](https://man.openbsd.org/OpenBSD-current/man8/sshd.8#AUTHORIZED_KEYS_FILE_FORMAT) on ssh settings to protect the system in the event of account compromise. - -### Setting up PTRACK Backups - -Backup mode PTACK can be used only on Postgrespro Standart and Postgrespro Enterprise installations or patched vanilla PostgreSQL. Links to ptrack patches can be found [here](https://github.com/postgrespro/pg_probackup#ptrack-support). - -If you are going to use PTRACK backups, complete the following additional steps: - -- Set the parameter `ptrack_enable` to `on`. -- Grant the rights to execute `ptrack` functions to the *backup* role **in every database** of the cluster: - - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; - GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; - -- The *backup* role must have access to all the databases of the cluster. - -## Usage - -### Creating a Backup - -To create a backup, run the following command: - - pg_probackup backup -B backup_dir --instance instance_name -b backup_mode - -Where *backup_mode* can take one of the following values: - -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have generated since the previous full or incremental backup was taken. Only changed blocks are readed from data files. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. - -When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the incremental backups between them, which is called `the backup chain`. You must create at least one full backup before taking incremental ones. - -#### ARCHIVE mode - -ARCHIVE is the default WAL delivery mode. - -For example, to make a FULL backup in ARCHIVE mode, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL - -Unlike backup in STREAM mode, ARCHIVE backup rely on [continuous archiving](#setting-up-continuous-wal-archiving) to provide WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - -During [backup](#backup) pg_probackup ensures that WAL files containing WAL records between START LSN and STOP LSN are actually exists in '*backup_dir*/wal/*instance_name*' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. - -#### STREAM mode - -STREAM is the optional WAL delivery mode. - -For example, to make a FULL backup in STREAM mode, add the `--stream` flag to the command from the previous example: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot - -The optional `--temp-slot` flag ensures that the required segments remain available if the WAL is rotated before the backup is complete. - -Unlike backup in ARCHIVE mode, STREAM backup include all the WAL segments required to restore the cluster to a consistent state at the time the backup was taken. - -During [backup](#backup) pg_probackup streams WAL files containing WAL records between START LSN and STOP LSN to '*backup_dir*/backups/*instance_name*/*BACKUP ID*/database/pg_wal' directory. Also pg_probackup ensures that WAL records between START LSN and STOP LSN can be parsed. This precations eliminates the risk of silent WAL corruption. - -Even if you are using [continuous archiving](#setting-up-continuous-wal-archiving), STREAM backups can still be useful in the following cases: - -- STREAM backups can be restored on the server that has no file access to WAL archive. -- STREAM backups enable you to restore the cluster state at the point in time for which WAL files in archive are no longer available. -- Backup in STREAM mode can be taken from standby of a server, that generates small amount of WAL traffic, without long waiting for WAL segment to fill up. - -#### Page validation - -If [data checksums](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS) are enabled in the database cluster, pg_probackup uses this information to check correctness of data files during backup. While reading each page, pg_probackup checks whether the calculated checksum coincides with the checksum stored in the page header. This guarantees that the PostgreSQL instance and backup itself are free of corrupted pages. -Note that pg_probackup reads database files directly from filesystem, so under heavy write load during backup it can show false positive checksum failures because of partial writes. In case of page checksumm mismatch, page is readed again and checksumm comparison repeated. - -Page is considered corrupted if checksumm comparison failed more than 100 times, in this case backup is aborted. - -Redardless of data checksums been enabled or not, pg_probackup always check page header "sanity". - -#### External directories - -To back up a directory located outside of the data directory, use the optional `--external-dirs` parameter that specifies the path to this directory. If you would like to add more than one external directory, provide several paths separated by colons, on Windows system paths must be separated by semicolon instead. - -For example, to include `'/etc/dir1/'` and `'/etc/dir2/'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 - -For example, to include `'C:\dir1\'` and `'C:\dir2\'` directories into the full backup of your *instance_name* instance that will be stored under the *backup_dir* directory on Windows system, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 - -pg_probackup creates a separate subdirectory in the backup directory for each external directory. Since external directories included into different backups do not have to be the same, when you are restoring the cluster from an incremental backup, only those directories that belong to this particular backup will be restored. Any external directories stored in the previous backups will be ignored. - -To include the same directories into each backup of your instance, you can specify them in the pg_probackup.conf configuration file using the [set-config](#set-config) command with the `--external-dirs` option. - -### Verifying a Cluster - -To verify that PostgreSQL database cluster is free of corruption, run the following command: - - pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] - -This physical verification works similar to [page validation](#page-validation) that is done during backup with several differences: - -- `checkdb` is read-only -- if corrupted page is detected, `checkdb` is not aborted, but carry on, until all pages in the cluster are validated -- `checkdb` do not strictly require *the backup catalog*, so it can be used to verify database clusters that are **not** [added to the backup catalog](#adding-a-new-backup-instance). - -If *backup_dir* and *instance_name* are omitted, then [connection options](#connection-options) and *data_dir* must be provided via environment variables or command-line options. - -Physical verification cannot detect logical inconsistencies, missing and nullified blocks or entire files, repercussions from PostgreSQL bugs and other wicked anomalies. -Extensions [amcheck](https://www.postgresql.org/docs/current/amcheck.html) and [amcheck_next](https://github.com/petergeoghegan/amcheck) provide a partial solution to these problems. - -If you would like, in addition to physical verification, to verify all indexes in all databases using these extensions, you can specify `--amcheck` flag when running [checkdb](#checkdb) command: - - pg_probackup checkdb -D data_dir --amcheck - -Physical verification can be skipped if `--skip-block-validation` flag is used. For logical only verification *backup_dir* and *data_dir* are optional, only [connection options](#connection-options) are mandatory: - - pg_probackup checkdb --amcheck --skip-block-validation {connection_options} - -Logical verification can be done more thoroughly with flag `--heapallindexed` by checking that all heap tuples that should be indexed are actually indexed, but at the higher cost of CPU, memory and I/O comsumption. - -### Validating a Backup - -pg_probackup calculates checksums for each file in a backup during backup process. The process of checking checksumms of backup data files is called `the backup validation`. By default validation is run immediately after backup is taken and right before restore, to detect possible backup corruption. - -If you would like to skip backup validation, you can specify the `--no-validate` flag when running [backup](#backup) and [restore](#restore) commands. - -To ensure that all the required backup files are present and can be used to restore the database cluster, you can run the [validate](#validate) command with the exact [recovery target options](#recovery-target-options) you are going to use for recovery. - -For example, to check that you can restore the database cluster from a backup copy up to the specified xid transaction ID, run this command: - - pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 - -If validation completes successfully, pg_probackup displays the corresponding message. If validation fails, you will receive an error message with the exact time, transaction ID and LSN up to which the recovery is possible. - -If you specify *backup_id* via `-i/--backup-id` option, then only backup copy with specified backup ID will be validated. If *backup_id* is specified with [recovery target options](#recovery-target-options) then validate will check whether it is possible to restore the specified backup to the specified `recovery target`. - -For example, to check that you can restore the database cluster from a backup copy with *backup_id* up to the specified timestamp, run this command: - - pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' - -If *backup_id* belong to incremental backup, then all its parents starting from FULL backup will be validated. - -If you omit all the parameters, all backups are validated. - -### Restoring a Cluster - -To restore the database cluster from a backup, run the restore command with at least the following options: - - pg_probackup restore -B backup_dir --instance instance_name -i backup_id - -Where: - -- *backup_dir* is the backup catalog that stores all backup files and meta information. -- *instance_name* is the backup instance for the cluster to be restored. -- *backup_id* specifies the backup to restore the cluster from. If you omit this option, pg_probackup uses the latest valid backup available for the specified instance. If you specify an incremental backup to restore, pg_probackup automatically restores the underlying full backup and then sequentially applies all the necessary increments. - -If the cluster to restore contains tablespaces, pg_probackup restores them to their original location by default. To restore tablespaces to a different location, use the `--tablespace-mapping/-T` option. Otherwise, restoring the cluster on the same host will fail if tablespaces are in use, because the backup would have to be written to the same directories. - -When using the `--tablespace-mapping/-T` option, you must provide absolute paths to the old and new tablespace directories. If a path happens to contain an equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. For example: - - pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir - -Once the restore command is complete, start the database service. - -If you are restoring an STREAM backup, the restore is complete at once, with the cluster returned to a self-consistent state at the point when the backup was taken. For ARCHIVE backups, PostgreSQL replays all available archived WAL segments, so the cluster is restored to the latest state possible. You can change this behavior by using the [recovery target options](#recovery-target-options) with the `restore` command. Note that using the [recovery target options](#recovery-target-options) when restoring STREAM backup is possible if the WAL archive is available at least starting from the time the STREAM backup was taken. - -To restore cluster on remote host see the section [Using pg_probackup in the Remote Mode](#using-pg-probackup-in-the-remote-mode). - ->NOTE: By default, the [restore](#restore) command validates the specified backup before restoring the cluster. If you run regular backup validations and would like to save time when restoring the cluster, you can specify the `--no-validate` flag to skip validation and speed up the recovery. - -#### Partial Restore - -If you have enabled [partial restore](#setting-up-partial-restore) before taking backups, you can restore or exclude from restore the arbitrary number of specific databases using [partial restore options](#partial-restore-options) with the [restore](#restore) commands. - -To restore only one or more databases, run the restore command with the following options: - - pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name - -The option `--db-include` can be specified multiple times. For example, to restore only databases `db1` and `db2`, run the following command: - - pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 - -To exclude one or more specific databases from restore, run the following options: - - pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name - -The option `--db-exclude` can be specified multiple times. For example, to exclude the databases `db1` and `db2` from restore, run the following command: - - pg_probackup restore -B backup_dir --instance instance_name -i backup_id --db-exclude=db1 --db-exclude=db2 - -Partial restore rely on lax behaviour of PostgreSQL recovery process toward truncated files. Files of excluded databases restored as null sized files, allowing recovery to work properly. After successfull starting of PostgreSQL cluster, you must drop excluded databases using `DROP DATABASE` command. - ->NOTE: The databases `template0` and `template1` are always restored. - -### Performing Point-in-Time (PITR) Recovery - -If you have enabled [continuous WAL archiving](#setting-up-continuous-wal-archiving) before taking backups, you can restore the cluster to its state at an arbitrary point in time (recovery target) using [recovery target options](#recovery-target-options) with the [restore](#restore) and [validate](#validate) commands. - -If `-i/--backup-id` option is omitted, pg_probackup automatically chooses the backup that is the closest to the specified recovery target and starts the restore process, otherwise pg_probackup will try to restore *backup_id* to the specified recovery target. - -- To restore the cluster state at the exact time, specify the `--recovery-target-time` option, in the timestamp format. For example: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' - -- To restore the cluster state up to a specific transaction ID, use the `--recovery-target-xid` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 - -- To restore the cluster state up to a specific LSN, use `--recovery-target-lsn` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 - -- To restore the cluster state up to a specific named restore point, use `--recovery-target-name` option: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' - -- To restore the backup to the latest state available in archive, use `--recovery-target` option with `latest` value: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' - -- To restore the cluster to the earliest point of consistency, use `--recovery-target` option with `immediate` value: - - pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' - -### Using pg_probackup in the Remote Mode - -pg_probackup supports the remote mode that allows to perform `backup` and `restore` operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed up is located on a remote system. You must have pg_probackup installed on both systems. - -Do note that pg_probackup rely on passwordless SSH connection for communication between the hosts. - -The typical workflow is as follows: - - - On your backup host, configure pg_probackup as explained in the section [Installation and Setup](#installation-and-setup). For the [add-instance](#add-instance) and [set-config](#set-config) commands, make sure to specify [remote options](#remote-mode-options) that point to the database host with the PostgreSQL instance. - -- If you would like to take remote backup in [PAGE](#creating-a-backup) mode, or rely on [ARCHIVE](#archive-mode) WAL delivery mode, or use [PITR](#performing-point-in-time-pitr-recovery), then configure continuous WAL archiving from database host to the backup host as explained in the section [Setting up continuous WAL archiving](#setting-up-continuous-wal-archiving). For the [archive-push](#archive-push) and [archive-get](#archive-get) commands, you must specify the [remote options](#remote-mode-options) that point to backup host with backup catalog. - -- Run [backup](#backup) or [restore](#restore) commands with [remote options](#remote-mode-options) **on backup host**. pg_probackup connects to the remote system via SSH and creates a backup locally or restores the previously taken backup on the remote system, respectively. - -For example, to create archive full backup using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - -For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302`, run: - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 - -Restoring of ARCHIVE backup or performing PITR in remote mode require additional information: destination address, port and username for establishing ssh connection **from** a host with database **to** a host with backup catalog. This information will be used by `restore_command` to copy via ssh WAL segments from archive to PostgreSQL 'pg_wal' directory. - -To solve this problem you can use [Remote Wal Archive Options](#remote-wal-archive-options). - -For example, to restore latest backup on remote system using remote mode through SSH connection to user `postgres` on host with address `192.168.0.2` via port `2302` and user `backup` on backup catalog host with address `192.168.0.3` via port `2303`, run: - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup - -Provided arguments will be used to construct 'restore_command' in recovery.conf: -``` -# recovery.conf generated by pg_probackup 2.1.5 -restore_command = 'pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' -``` - -Alternatively you can use `--restore-command` option to provide an entire 'restore_command': - - pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' - ->NOTE: The remote backup mode is currently unavailable for Windows systems. - -### Running pg_probackup on Parallel Threads - -[Backup](#backup), [restore](#restore), [merge](#merge), [delete](#delete), [checkdb](#checkdb) and [validate](#validate) processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU cores, disk and network bandwidth). - -Parallel execution is controlled by the `-j/--threads` command line option. For example, to create a backup using four parallel threads, run: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 - ->NOTE: Parallel restore applies only to copying data from the backup catalog to the data directory of the cluster. When PostgreSQL server is started, WAL records need to be replayed, and this cannot be done in parallel. - -### Configuring pg_probackup - -Once the backup catalog is initialized and a new backup instance is added, you can use the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory to fine-tune pg_probackup configuration. - -For example, [backup](#backup) and [checkdb](#checkdb) commands uses a regular PostgreSQL connection. To avoid specifying these options each time on the command line, you can set them in the pg_probackup.conf configuration file using the [set-config](#set-config) command. - ->NOTE: It is **not recommended** to edit pg_probackup.conf manually. - -Initially, pg_probackup.conf contains the following settings: - -- PGDATA — the path to the data directory of the cluster to back up. -- system-identifier — the unique identifier of the PostgreSQL instance. - -Additionally, you can define [remote](#remote-mode-options), [retention](#retention-options), [logging](#logging-options) and [compression](#compression-options) settings using the `set-config` command: - - pg_probackup set-config -B backup_dir --instance instance_name - [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] - -To view the current settings, run the following command: - - pg_probackup show-config -B backup_dir --instance instance_name - -You can override the settings defined in pg_probackup.conf when running pg_probackups [commands](#commands) via corresponding environment variables and/or command line options. - -### Specifying Connection Settings - -If you define connection settings in the 'pg_probackup.conf' configuration file, you can omit connection options in all the subsequent pg_probackup commands. However, if the corresponding environment variables are set, they get higher priority. The options provided on the command line overwrite both environment variables and configuration file settings. - -If nothing is given, the default values are taken. By default pg_probackup tries to use local connection via Unix domain socket (localhost on Windows) and tries to get the database name and the user name from the PGUSER environment variable or the current OS user name. - -### Managing the Backup Catalog - -With pg_probackup, you can manage backups from the command line: - -- [View backup information](#viewing-backup-information) -- [View WAL Archive Information](#viewing-wal-archive-information) -- [Validate backups](#validating-a-backup) -- [Merge backups](#merging-backups) -- [Delete backups](#deleting-backups) - -#### Viewing Backup Information - -To view the list of existing backups for every instance, run the command: - - pg_probackup show -B backup_dir - -pg_probackup displays the list of all the available backups. For example: - -``` -BACKUP INSTANCE 'node' -====================================================================================================================================== - Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -====================================================================================================================================== - node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK -``` - -For each backup, the following information is provided: - -- Instance — the instance name. -- Version — PostgreSQL major version. -- ID — the backup identifier. -- Recovery time — the earliest moment for which you can restore the state of the database cluster. -- Mode — the method used to take this backup. Possible values: FULL, PAGE, DELTA, PTRACK. -- WAL Mode — the WAL delivery mode. Possible values: STREAM and ARCHIVE. -- TLI — timeline identifiers of current backup and its parent. -- Time — the time it took to perform the backup. -- Data — the size of the data files in this backup. This value does not include the size of WAL files. In case of STREAM backup the total size of backup can be calculated as 'Data' + 'WAL'. -- WAL — the uncompressed size of WAL files required to apply by PostgreSQL recovery process to reach consistency. -- Zratio — compression ratio calculated as 'uncompressed-bytes' / 'data-bytes'. -- Start LSN — WAL log sequence number corresponding to the start of the backup process. REDO point for PostgreSQL recovery process to start from. -- Stop LSN — WAL log sequence number corresponding to the end of the backup process. Consistency point for PostgreSQL recovery process. -- Status — backup status. Possible values: - - - OK — the backup is complete and valid. - - DONE — the backup is complete, but was not validated. - - RUNNING — the backup is in progress. - - MERGING — the backup is being merged. - - DELETING — the backup files are being deleted. - - CORRUPT — some of the backup files are corrupted. - - ERROR — the backup was aborted because of an unexpected error. - - ORPHAN — the backup is invalid because one of its parent backups is corrupt or missing. - -You can restore the cluster from the backup only if the backup status is OK or DONE. - -To get more detailed information about the backup, run the show with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name -i backup_id - -The sample output is as follows: - -``` -#Configuration -backup-mode = FULL -stream = false -compress-alg = zlib -compress-level = 1 -from-replica = false - -#Compatibility -block-size = 8192 -wal-block-size = 8192 -checksum-version = 1 -program-version = 2.1.3 -server-version = 10 - -#Result backup info -timelineid = 1 -start-lsn = 0/04000028 -stop-lsn = 0/040000f8 -start-time = '2017-05-16 12:57:29' -end-time = '2017-05-16 12:57:31' -recovery-xid = 597 -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-05-16 12:57:31' -data-bytes = 22288792 -wal-bytes = 16777216 -uncompressed-bytes = 39961833 -pgdata-bytes = 39859393 -status = OK -parent-backup-id = 'PT8XFX' -primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' -``` - -Detailed output has additional attributes: -- compress-alg — compression algorithm used during backup. Possible values: 'zlib', 'pglz', 'none'. -- compress-level — compression level used during backup. -- from-replica — the fact that backup was taken from standby server. Possible values: '1', '0'. -- block-size — (block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. -- wal-block-size — (wal_block_size)[https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-WAL-BLOCK-SIZE] setting of PostgreSQL cluster at the moment of backup start. -- checksum-version — the fact that PostgreSQL cluster, from which backup is taken, has enabled [data block checksumms](https://www.postgresql.org/docs/current/runtime-config-preset.html#GUC-DATA-CHECKSUMS). Possible values: '1', '0'. -- program-version — full version of pg_probackup binary used to create backup. -- start-time — the backup starting time. -- end-time — the backup ending time. -- expire-time — if the backup was pinned, then until this point in time the backup cannot be removed by retention purge. -- uncompressed-bytes — size of the data files before adding page headers and applying compression. You can evaluate the effectiveness of compression by comparing 'uncompressed-bytes' to 'data-bytes' if compression if used. -- pgdata-bytes — size of the PostgreSQL cluster data files at the time of backup. You can evaluate the effectiveness of incremental backup by comparing 'pgdata-bytes' to 'uncompressed-bytes'. -- recovery-xid — current transaction id at the moment of backup ending. -- parent-backup-id — backup ID of parent backup. Available only for incremental backups. -- primary_conninfo — libpq conninfo used for connection to PostgreSQL cluster during backup. The password is not included. - -To get more detailed information about the backup in json format, run the show with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id - -The sample output is as follows: - -``` -[ - { - "instance": "node", - "backups": [ - { - "id": "PT91HZ", - "parent-backup-id": "PT8XFX", - "backup-mode": "DELTA", - "wal": "ARCHIVE", - "compress-alg": "zlib", - "compress-level": 1, - "from-replica": false, - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.3", - "server-version": "10", - "current-tli": 16, - "parent-tli": 2, - "start-lsn": "0/8000028", - "stop-lsn": "0/8000160", - "start-time": "2019-06-17 18:25:11+03", - "end-time": "2019-06-17 18:25:16+03", - "recovery-xid": 0, - "recovery-time": "2019-06-17 18:25:15+03", - "data-bytes": 106733, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } -] -``` - -#### Viewing WAL Archive Information - -To view the information about WAL archive for every instance, run the command: - - pg_probackup show -B backup_dir [--instance instance_name] --archive - -pg_probackup displays the list of all the available WAL files grouped by timelines. For example: - -``` -ARCHIVE INSTANCE 'node' -=================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=================================================================================================================== - 5 1 0/B000000 000000000000000B 000000000000000C 2 685kB 48.00 0 OK - 4 3 0/18000000 0000000000000018 000000000000001A 3 648kB 77.00 0 OK - 3 2 0/15000000 0000000000000015 0000000000000017 3 648kB 77.00 0 OK - 2 1 0/B000108 000000000000000B 0000000000000015 5 892kB 94.00 1 DEGRADED - 1 0 0/0 0000000000000001 000000000000000A 10 8774kB 19.00 1 OK - -``` - -For each backup, the following information is provided: - -- TLI — timeline identifier. -- Parent TLI — identifier of timeline TLI branched off. -- Switchpoint — LSN of the moment when the timeline branched off from "Parent TLI". -- Min Segno — number of the first existing WAL segment belonging to the timeline. -- Max Segno — number of the last existing WAL segment belonging to the timeline. -- N segments — number of WAL segments belonging to the timeline. -- Size — the size files take on disk. -- Zratio — compression ratio calculated as 'N segments' * wal_seg_size / 'Size'. -- N backups — number of backups belonging to the timeline. To get the details about backups, use json format. -- Status — archive status for this exact timeline. Possible values: - - OK — all WAL segments between Min and Max are present. - - DEGRADED — some WAL segments between Min and Max are lost. To get details about lost files, use json format. - -To get more detailed information about the WAL archive in json format, run the command: - - pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json - -The sample output is as follows: - -``` -[ - { - "instance": "replica", - "timelines": [ - { - "tli": 5, - "parent-tli": 1, - "switchpoint": "0/B000000", - "min-segno": "000000000000000B", - "max-segno": "000000000000000C", - "n-segments": 2, - "size": 685320, - "zratio": 48.00, - "closest-backup-id": "PXS92O", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 4, - "parent-tli": 3, - "switchpoint": "0/18000000", - "min-segno": "0000000000000018", - "max-segno": "000000000000001A", - "n-segments": 3, - "size": 648625, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 3, - "parent-tli": 2, - "switchpoint": "0/15000000", - "min-segno": "0000000000000015", - "max-segno": "0000000000000017", - "n-segments": 3, - "size": 648911, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 2, - "parent-tli": 1, - "switchpoint": "0/B000108", - "min-segno": "000000000000000B", - "max-segno": "0000000000000015", - "n-segments": 5, - "size": 892173, - "zratio": 94.00, - "closest-backup-id": "PXS92O", - "status": "DEGRADED", - "lost-segments": [ - { - "begin-segno": "000000000000000D", - "end-segno": "000000000000000E" - }, - { - "begin-segno": "0000000000000010", - "end-segno": "0000000000000012" - } - ], - "backups": [ - { - "id": "PXS9CE", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 2, - "parent-tli": 0, - "start-lsn": "0/C000028", - "stop-lsn": "0/C000160", - "start-time": "2019-09-13 21:43:26+03", - "end-time": "2019-09-13 21:43:30+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:43:29+03", - "data-bytes": 104674852, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - }, - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "0000000000000001", - "max-segno": "000000000000000A", - "n-segments": 10, - "size": 8774805, - "zratio": 19.00, - "closest-backup-id": "", - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92O", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "true", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/4000028", - "stop-lsn": "0/6000028", - "start-time": "2019-09-13 21:37:36+03", - "end-time": "2019-09-13 21:38:45+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 25987319, - "wal-bytes": 50331648, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - }, - { - "instance": "master", - "timelines": [ - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "0000000000000001", - "max-segno": "000000000000000B", - "n-segments": 11, - "size": 8860892, - "zratio": 20.00, - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92H", - "parent-backup-id": "PXS92C", - "backup-mode": "PAGE", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 1, - "start-lsn": "0/4000028", - "stop-lsn": "0/50000B8", - "start-time": "2019-09-13 21:37:29+03", - "end-time": "2019-09-13 21:37:31+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 1328461, - "wal-bytes": 33554432, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - }, - { - "id": "PXS92C", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/2000028", - "stop-lsn": "0/2000160", - "start-time": "2019-09-13 21:37:24+03", - "end-time": "2019-09-13 21:37:29+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:28+03", - "data-bytes": 24871902, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - } -] -``` - -Most fields are consistent with plain format, with some exceptions: - -- size is in bytes. -- 'closest-backup-id' attribute contain ID of valid backup closest to the timeline, located on some of the previous timelines. This backup is the closest starting point to reach the timeline from other timelines by PITR. Closest backup always has a valid status, either OK or DONE. If such backup do not exists, then string is empty. -- DEGRADED timelines contain 'lost-segments' array with information about intervals of missing segments. In OK timelines 'lost-segments' array is empty. -- 'N backups' attribute is replaced with 'backups' array containing backups belonging to the timeline. If timeline has no backups, then 'backups' array is empty. - -### Configuring Retention Policy - -With pg_probackup, you can set retention policies for backups and WAL archive. All policies can be combined together in any way. - -#### Backup Retention Policy - -By default, all backup copies created with pg_probackup are stored in the specified backup catalog. To save disk space, you can configure retention policy and periodically clean up redundant backup copies accordingly. - -To configure retention policy, set one or more of the following variables in the pg_probackup.conf file via [set-config](#set-config): - - --retention-redundancy=redundancy -Specifies **the number of full backup copies** to keep in the backup catalog. - - --retention-window=window -Defines the earliest point in time for which pg_probackup can complete the recovery. This option is set in **the number of days** from the current moment. For example, if `retention-window=7`, pg_probackup must delete all backup copies that are older than seven days, with all the corresponding WAL files. - -If both `--retention-redundancy` and `--retention-window` options are set, pg_probackup keeps backup copies that satisfy at least one condition. For example, if you set `--retention-redundancy=2` and `--retention-window=7`, pg_probackup purges the backup catalog to keep only two full backup copies and all backups that are newer than seven days: - - pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 - -To clean up the backup catalog in accordance with retention policy, run: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired - -pg_probackup deletes all backup copies that do not conform to the defined retention policy. - -If you would like to also remove the WAL files that are no longer required for any of the backups, add the `--delete-wal` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal - ->NOTE: Alternatively, you can use the `--delete-expired`, `--merge-expired`, `--delete-wal` flags and the `--retention-window` and `--retention-redundancy` options together with the [backup](#backup) command to remove and merge the outdated backup copies once the new backup is created. - -You can set or override the current retention policy by specifying `--retention-redundancy` and `--retention-window` options directly when running `delete` or `backup` commands: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 - -Since incremental backups require that their parent full backup and all the preceding incremental backups are available, if any of such backups expire, they still cannot be removed while at least one incremental backup in this chain satisfies the retention policy. To avoid keeping expired backups that are still required to restore an active incremental one, you can merge them with this backup using the `--merge-expired` flag when running [backup](#backup) or [delete](#delete) commands. - -Suppose you have backed up the *node* instance in the *backup_dir* directory, with the `--retention-window` option is set to *7*, and you have the following backups available on April 10, 2019: - -``` -BACKUP INSTANCE 'node' -=================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -=================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK - -------------------------------------------------------retention window-------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK -``` - -Even though P7XDHB and P7XDHU backups are outside the retention window, they cannot be removed as it invalidates the succeeding incremental backups P7XDJA and P7XDQV that are still required, so, if you run the [delete](#delete) command with the `--delete-expired` flag, only the P7XDFT full backup will be removed. - -With the `--merge-expired` option, the P7XDJA backup is merged with the underlying P7XDHU and P7XDHB backups and becomes a full one, so there is no need to keep these expired backups anymore: - - pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired - pg_probackup show -B backup_dir - -``` -BACKUP INSTANCE 'node' -================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK -``` - ->NOTE: The Time field for the merged backup displays the time required for the merge. - -#### Backup Pinning - -If you have the necessity to exclude certain backups from established retention policy then it is possible to pin a backup for an arbitrary amount of time. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d - -This command will set `expire-time` of the specified backup to 30 days starting from backup `recovery-time` attribute. Basically `expire-time` = `recovery-time` + `ttl`. - -Also you can set `expire-time` explicitly using `--expire-time` option. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' - -Alternatively you can use the `--ttl` and `--expire-time` options with the [backup](#backup) command to pin newly created backup: - - pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d - pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' - -You can determine the fact that backup is pinned and check due expire time by looking up `expire-time` attribute in backup metadata via [show](#show) command: - - pg_probackup show --instance instance_name -i backup_id - -Pinned backup has `expire-time` attribute: -``` -... -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-01-01 00:00:00+03' -data-bytes = 22288792 -... -``` - -You can unpin the backup by setting the `--ttl` option to zero using [set-backup](#set-backup) command. Example: - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 - -Only pinned backups have the `expire-time` attribute in the backup metadata. - ->NOTE: Pinned incremental backup will also implicitly pin all its parent backups. - -#### WAL Archive Retention Policy - -By default, pg_probackup treatment of WAL Archive is very conservative and only "redundant" WAL segments can be purged, i.e. segments that cannot be applied to any of the existing backups in the backup catalog. To save disk space, you can configure WAL Archive retention policy, that allows to keep WAL of limited depth measured in backups per timeline. - -Suppose you have backed up the *node* instance in the *backup_dir* directory with configured [WAL archiving](#setting-up-continuous-wal-archiving): - - pg_probackup show -B backup_dir --instance node - -``` -BACKUP INSTANCE 'node' -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK - node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK -``` - -The state of WAL archive can be determined by using [show](#command) command with `--archive` flag: - - pg_probackup show -B backup_dir --instance node --archive - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000001 0000000000000047 71 36MB 31.00 6 OK -``` - -General WAL purge without `wal-depth` cannot achieve much, only one segment can be removed: - - pg_probackup delete -B backup_dir --instance node --delete-wal - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000002 0000000000000047 70 34MB 32.00 6 OK -``` - -If you would like, for example, to keep only those WAL segments that can be applied to the last valid backup, use the `--wal-depth` option: - - pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 - -``` -ARCHIVE INSTANCE 'node' -================================================================================================================ - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -================================================================================================================ - 1 0 0/0 0000000000000046 0000000000000047 2 143kB 228.00 6 OK -``` - -Alternatively you can use the `--wal-depth` option with the [backup](#backup) command: - - pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal - -``` -ARCHIVE INSTANCE 'node' -=============================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================== - 1 0 0/0 0000000000000048 0000000000000049 1 72kB 228.00 7 OK -``` - -### Merging Backups - -As you take more and more incremental backups, the total size of the backup catalog can substantially grow. To save disk space, you can merge incremental backups to their parent full backup by running the merge command, specifying the backup ID of the most recent incremental backup you would like to merge: - - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - -This command merges the specified incremental backup to its parent full backup, together with all incremental backups between them. Once the merge is complete, the incremental backups are removed as redundant. Thus, the merge operation is virtually equivalent to retaking a full backup and removing all the outdated backups, but it allows to save much time, especially for large data volumes, I/O and network traffic in case of [remote](#using-pg_probackup-in-the-remote-mode) backup. - -Before the merge, pg_probackup validates all the affected backups to ensure that they are valid. You can check the current backup status by running the [show](#show) command with the backup ID: - - pg_probackup show -B backup_dir --instance instance_name -i backup_id - -If the merge is still in progress, the backup status is displayed as MERGING. The merge is idempotent, so you can restart the merge if it was interrupted. - -### Deleting Backups - -To delete a backup that is no longer required, run the following command: - - pg_probackup delete -B backup_dir --instance instance_name -i backup_id - -This command will delete the backup with the specified *backup_id*, together with all the incremental backups that descend from *backup_id* if any. This way you can delete some recent incremental backups, retaining the underlying full backup and some of the incremental backups that follow it. - -To delete obsolete WAL files that are not necessary to restore any of the remaining backups, use the `--delete-wal` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-wal - -To delete backups that are expired according to the current retention policy, use the `--delete-expired` flag: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired - -Note that expired backups cannot be removed while at least one incremental backup that satisfies the retention policy is based on them. If you would like to minimize the number of backups still required to keep incremental backups valid, specify the `--merge-expired` flag when running this command: - - pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired - -In this case, pg_probackup searches for the oldest incremental backup that satisfies the retention policy and merges this backup with the underlying full and incremental backups that have already expired, thus making it a full backup. Once the merge is complete, the remaining expired backups are deleted. - -Before merging or deleting backups, you can run the `delete` command with the `--dry-run` flag, which displays the status of all the available backups according to the current retention policy, without performing any irreversible actions. - -## Command-Line Reference -### Commands - -This section describes pg_probackup commands. Some commands require mandatory parameters and can take additional options. Optional parameters encased in square brackets. For detailed descriptions of options, see the section [Options](#options). - -#### version - - pg_probackup version - -Prints pg_probackup version. - -#### help - - pg_probackup help [command] - -Displays the synopsis of pg_probackup commands. If one of the pg_probackup commands is specified, shows detailed information about the options that can be used with this command. - -#### init - - pg_probackup init -B backup_dir [--help] - -Initializes the backup catalog in *backup_dir* that will store backup copies, WAL archive and meta information for the backed up database clusters. If the specified *backup_dir* already exists, it must be empty. Otherwise, pg_probackup displays a corresponding error message. - -For details, see the secion [Initializing the Backup Catalog](#initializing-the-backup-catalog). - -#### add-instance - - pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name - [--help] - -Initializes a new backup instance inside the backup catalog *backup_dir* and generates the pg_probackup.conf configuration file that controls pg_probackup settings for the cluster with the specified *data_dir* data directory. - -For details, see the section [Adding a New Backup Instance](#adding-a-new-backup-instance). - -#### del-instance - - pg_probackup del-instance -B backup_dir --instance instance_name - [--help] - -Deletes all backups and WAL files associated with the specified instance. - -#### set-config - - pg_probackup set-config -B backup_dir --instance instance_name - [--help] [--pgdata=pgdata-path] - [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] - [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] - [-d dbname] [-h host] [-p port] [-U username] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [--restore-command=cmdline] - [remote_options] [remote_archive_options] [logging_options] - -Adds the specified connection, compression, retention, logging and external directory settings into the pg_probackup.conf configuration file, or modifies the previously defined values. - -For all available settings, see the [Options](#options) section. - -It is **not recommended** to edit pg_probackup.conf manually. - -#### set-backup - - pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id - {--ttl=ttl | --expire-time=time} [--help] - -Sets the provided backup-specific settings into the backup.control configuration file, or modifies previously defined values. - -For all available settings, see the section [Pinning Options](#pinning-options). - -#### show-config - - pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] - -Displays the contents of the pg_probackup.conf configuration file located in the '*backup_dir*/backups/*instance_name*' directory. You can specify the `--format=json` option to return the result in the JSON format. By default, configuration settings are shown as plain text. - -To edit pg_probackup.conf, use the [set-config](#set-config) command. - -#### show - - pg_probackup show -B backup_dir - [--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] - -Shows the contents of the backup catalog. If *instance_name* and *backup_id* are specified, shows detailed information about this backup. You can specify the `--format=json` option to return the result in the JSON format. If `--archive` option is specified, shows the content of WAL archive of the backup catalog. - -By default, the contents of the backup catalog is shown as plain text. - -For details on usage, see the sections [Managing the Backup Catalog](#managing-the-backup-catalog) and [Viewing WAL Archive Information](#viewing-wal-archive-information). - - -#### backup - - pg_probackup backup -B backup_dir -b backup_mode --instance instance_name - [--help] [-j num_threads] [--progress] - [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] - [--no-validate] [--skip-block-validation] - [-w --no-password] [-W --password] - [--archive-timeout=timeout] [--external-dirs=external_directory_path] - [connection_options] [compression_options] [remote_options] - [retention_options] [pinning_options] [logging_options] - -Creates a backup copy of the PostgreSQL instance. The *backup_mode* option specifies the backup mode to use. - - -b mode - --backup-mode=mode - -Specifies the backup mode to use. Possible values are: - -- FULL — creates a full backup that contains all the data files of the cluster to be restored. -- DELTA — reads all data files in the data directory and creates an incremental backup for pages that have changed since the previous backup. -- PAGE — creates an incremental PAGE backup based on the WAL files that have changed since the previous full or incremental backup was taken. -- PTRACK — creates an incremental PTRACK backup tracking page changes on the fly. - -``` --C ---smooth-checkpoint -``` -Spreads out the checkpoint over a period of time. By default, pg_probackup tries to complete the checkpoint as soon as possible. - - --stream -Makes an [STREAM](#stream-mode) backup that includes all the necessary WAL files by streaming them from the database server via replication protocol. - - --temp-slot -Creates a temporary physical replication slot for streaming WAL from the backed up PostgreSQL instance. It ensures that all the required WAL segments remain available if WAL is rotated while the backup is in progress. This flag can only be used together with the `--stream` flag. Default slot name is `pg_probackup_slot`, which can be changed via option `--slot/-S`. - - -S slot_name - --slot=slot_name -Specifies the replication slot for WAL streaming. This option can only be used together with the `--stream` flag. - - --backup-pg-log -Includes the log directory into the backup. This directory usually contains log messages. By default, log directory is excluded. - - -E external_directory_path - --external-dirs=external_directory_path -Includes the specified directory into the backup. This option is useful to back up scripts, sql dumps and configuration files located outside of the data directory. If you would like to back up several external directories, separate their paths by a colon on Unix and a semicolon on Windows. - - --archive-timeout=wait_time -Sets in seconds the timeout for WAL segment archiving and streaming. By default pg_probackup waits 300 seconds. - - --skip-block-validation -Disables block-level checksum verification to speed up backup. - - --no-validate -Skips automatic validation after successfull backup. You can use this flag if you validate backups regularly and would like to save time when running backup operations. - -Additionally [Connection Options](#connection-options), [Retention Options](#retention-options), [Pinning Options](#pinning-options), [Remote Mode Options](#remote-mode-options), [Compression Options](#compression-options), [Logging Options](#logging-options) and [Common Options](#common-options) can be used. - -For details on usage, see the section [Creating a Backup](#creating-a-backup). - -#### restore - - pg_probackup restore -B backup_dir --instance instance_name - [--help] [-D data_dir] [-i backup_id] - [-j num_threads] [--progress] - [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] - [-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force] - [--restore-command=cmdline] - [recovery_options] [logging_options] [remote_options] - [partial_restore_options] [remote_archive_options] - -Restores the PostgreSQL instance from a backup copy located in the *backup_dir* backup catalog. If you specify a [recovery target option](#recovery-target-options), pg_probackup will find the closest backup and restores it to the specified recovery target. Otherwise, the most recent backup is used. - - -R | --restore-as-replica -Writes a minimal recovery.conf in the output directory to facilitate setting up a standby server. The password is not included. If the replication connection requires a password, you must specify the password manually. - - -T OLDDIR=NEWDIR - --tablespace-mapping=OLDDIR=NEWDIR - -Relocates the tablespace from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple tablespaces. - - --external-mapping=OLDDIR=NEWDIR -Relocates an external directory included into the backup from the OLDDIR to the NEWDIR directory at the time of recovery. Both OLDDIR and NEWDIR must be absolute paths. If the path contains the equals sign (=), escape it with a backslash. This option can be specified multiple times for multiple directories. - - --skip-external-dirs -Skip external directories included into the backup with the `--external-dirs` option. The contents of these directories will not be restored. - - --skip-block-validation -Disables block-level checksum verification to speed up validation. During automatic validation before restore only file-level checksums will be verified. - - --no-validate -Skips backup validation. You can use this flag if you validate backups regularly and would like to save time when running restore operations. - - --restore-command=cmdline -Set the [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) parameter to specified command. Example: `--restore-command='cp /mnt/server/archivedir/%f "%p"'` - - --force -Allows to ignore the invalid status of the backup. You can use this flag if you for some reason have the necessity to restore PostgreSQL cluster from corrupted or invalid backup. Use with caution. - -Additionally [Recovery Target Options](#recovery-target-options), [Remote Mode Options](#remote-mode-options), [Remote WAL Archive Options](#remote-wal-archive-options), [Logging Options](#logging-options), [Partial Restore](#partial-restore) and [Common Options](#common-options) can be used. - -For details on usage, see the section [Restoring a Cluster](#restoring-a-cluster). - -#### checkdb - - pg_probackup checkdb - [-B backup_dir] [--instance instance_name] [-D data_dir] - [--help] [-j num_threads] [--progress] - [--skip-block-validation] [--amcheck] [--heapallindexed] - [connection_options] [logging_options] - -Verifies the PostgreSQL database cluster correctness by detecting physical and logical corruption. - - --amcheck -Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking data files. You must have the `amcheck` extention or the `amcheck_next` extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. - - --skip-block-validation -Skip validation of data files. Can be used only with `--amcheck` flag, so only logical verification of indexes is performed. - - --heapallindexed -Checks that all heap tuples that should be indexed are actually indexed. You can use this flag only together with the `--amcheck` flag. Can be used only with `amcheck` extension of version 2.0 and `amcheck_next` extension of any version. - -Additionally [Connection Options](#connection-options) and [Logging Options](#logging-options) can be used. - -For details on usage, see the section [Verifying a Cluster](#verifying-a-cluster). - -#### validate - - pg_probackup validate -B backup_dir - [--help] [--instance instance_name] [-i backup_id] - [-j num_threads] [--progress] - [--skip-block-validation] - [recovery_target_options] [logging_options] - -Verifies that all the files required to restore the cluster are present and not corrupted. If *instance_name* is not specified, pg_probackup validates all backups available in the backup catalog. If you specify the *instance_name* without any additional options, pg_probackup validates all the backups available for this backup instance. If you specify the *instance_name* with a [recovery target options](#recovery-target-options) and/or a *backup_id*, pg_probackup checks whether it is possible to restore the cluster using these options. - -For details, see the section [Validating a Backup](#validating-a-backup). - -#### merge - - pg_probackup merge -B backup_dir --instance instance_name -i backup_id - [--help] [-j num_threads] [--progress] - [logging_options] - -Merges the specified incremental backup to its parent full backup, together with all incremental backups between them, if any. As a result, the full backup takes in all the merged data, and the incremental backups are removed as redundant. - -For details, see the section [Merging Backups](#merging-backups). - -#### delete - - pg_probackup delete -B backup_dir --instance instance_name - [--help] [-j num_threads] [--progress] - [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] - [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired} - [--dry-run] - [logging_options] - -Deletes backup with specified *backip_id* or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. - -For details, see the sections [Deleting Backups](#deleting-backups), [Retention Options](#retention-options) and [Configuring Retention Policy](#configuring-retention-policy). - -#### archive-push - - pg_probackup archive-push -B backup_dir --instance instance_name - --wal-file-path=wal_file_path --wal-file-name=wal_file_name - [--help] [--compress] [--compress-algorithm=compression_algorithm] - [--compress-level=compression_level] [--overwrite] - [remote_options] [logging_options] - -Copies WAL files into the corresponding subdirectory of the backup catalog and validates the backup instance by *instance_name* and *system-identifier*. If parameters of the backup instance and the cluster do not match, this command fails with the following error message: “Refuse to push WAL segment segment_name into archive. Instance parameters mismatch.” For each WAL file moved to the backup catalog, you will see the following message in PostgreSQL logfile: “pg_probackup archive-push completed successfully”. - -If the files to be copied already exist in the backup catalog, pg_probackup computes and compares their checksums. If the checksums match, archive-push skips the corresponding file and returns successful execution code. Otherwise, archive-push fails with an error. If you would like to replace WAL files in the case of checksum mismatch, run the archive-push command with the `--overwrite` flag. - -Copying is done to temporary file with `.part` suffix or, if [compression](#compression-options) is used, with `.gz.part` suffix. After copy is done, atomic rename is performed. This algorihtm ensures that failed archive-push will not stall continuous archiving and that concurrent archiving from multiple sources into single WAL archive has no risk of archive corruption. -Copied to archive WAL segments are synced to disk. - -You can use `archive-push` in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) PostgreSQL parameter to set up [continous WAl archiving](#setting-up-continuous-wal-archiving). - -For details, see sections [Archiving Options](#archiving-options) and [Compression Options](#compression-options). - -#### archive-get - - pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name - [--help] [remote_options] [logging_options] - -Copies WAL files from the corresponding subdirectory of the backup catalog to the cluster's write-ahead log location. This command is automatically set by pg_probackup as part of the `restore_command` in 'recovery.conf' when restoring backups using a WAL archive. You do not need to set it manually. - -### Options - -This section describes command-line options for pg_probackup commands. If the option value can be derived from an environment variable, this variable is specified below the command-line option, in the uppercase. Some values can be taken from the pg_probackup.conf configuration file located in the backup catalog. - -For details, see the section [Configuring pg_probackup](#configuring-pg_probackup). - -If an option is specified using more than one method, command-line input has the highest priority, while the pg_probackup.conf settings have the lowest priority. - -#### Common Options -The list of general options. - - -B directory - --backup-path=directory - BACKUP_PATH -Specifies the absolute path to the backup catalog. Backup catalog is a directory where all backup files and meta information are stored. Since this option is required for most of the pg_probackup commands, you are recommended to specify it once in the BACKUP_PATH environment variable. In this case, you do not need to use this option each time on the command line. - - -D directory - --pgdata=directory - PGDATA -Specifies the absolute path to the data directory of the database cluster. This option is mandatory only for the [add-instance](#add-instance) command. Other commands can take its value from the PGDATA environment variable, or from the pg_probackup.conf configuration file. - - -i backup_id - -backup-id=backup_id -Specifies the unique identifier of the backup. - - -j num_threads - --threads=num_threads -Sets the number of parallel threads for backup, restore, merge, validation and verification processes. - - --progress -Shows the progress of operations. - - --help -Shows detailed information about the options that can be used with this command. - -#### Recovery Target Options - -If [continuous WAL archiving](#setting-up-continuous-wal-archiving) is configured, you can use one of these options together with [restore](#restore) or [validate](#validate) commands to specify the moment up to which the database cluster must be restored or validated. - - --recovery-target=immediate|latest -Defines when to stop the recovery: - -- `immediate` value stops the recovery after reaching the consistent state of the specified backup, or the latest available backup if the `-i/--backup_id` option is omitted. -- `latest` value continues the recovery until all WAL segments available in the archive are applied. - -Default value of `--recovery-target` depends on WAL delivery method of restored backup, `immediate` for STREAM backup and `latest` for ARCHIVE. - - --recovery-target-timeline=timeline -Specifies a particular timeline to which recovery will proceed. By default, the timeline of the specified backup is used. - - --recovery-target-lsn=lsn -Specifies the LSN of the write-ahead log location up to which recovery will proceed. Can be used only when restoring database cluster of major version 10 or higher. - - --recovery-target-name=recovery_target_name -Specifies a named savepoint up to which to restore the cluster data. - - --recovery-target-time=time -Specifies the timestamp up to which recovery will proceed. - - --recovery-target-xid=xid -Specifies the transaction ID up to which recovery will proceed. - - --recovery-target-inclusive=boolean -Specifies whether to stop just after the specified recovery target (true), or just before the recovery target (false). This option can only be used together with `--recovery-target-name`, `--recovery-target-time`, `--recovery-target-lsn` or `--recovery-target-xid` options. The default depends on [recovery_target_inclusive](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-INCLUSIVE) parameter. - - --recovery-target-action=pause|promote|shutdown - Default: pause -Specifies [the action](https://www.postgresql.org/docs/current/recovery-target-settings.html#RECOVERY-TARGET-ACTION) the server should take when the recovery target is reached. - -#### Retention Options - -You can use these options together with [backup](#backup) and [delete](#delete) commands. - -For details on configuring retention policy, see the section [Configuring Retention Policy](#configuring-retention-policy). - - --retention-redundancy=redundancy - Default: 0 -Specifies the number of full backup copies to keep in the data directory. Must be a positive integer. The zero value disables this setting. - - --retention-window=window - Default: 0 -Number of days of recoverability. Must be a positive integer. The zero value disables this setting. - - --wal-depth=wal_depth - Default: 0 -Number of latest valid backups on every timeline that must retain the ability to perform PITR. Must be a positive integer. The zero value disables this setting. - - --delete-wal -Deletes WAL files that are no longer required to restore the cluster from any of the existing backups. - - --delete-expired -Deletes backups that do not conform to the retention policy defined in the pg_probackup.conf configuration file. - - --merge-expired -Merges the oldest incremental backup that satisfies the requirements of retention policy with its parent backups that have already expired. - - --dry-run -Displays the current status of all the available backups, without deleting or merging expired backups, if any. - -##### Pinning Options - -You can use these options together with [backup](#backup) and [set-delete](#set-backup) commands. - -For details on backup pinning, see the section [Backup Pinning](#backup-pinning). - - --ttl=ttl -Specifies the amount of time the backup should be pinned. Must be a positive integer. The zero value unpin already pinned backup. Supported units: ms, s, min, h, d (s by default). Example: `--ttl=30d`. - - --expire-time=time -Specifies the timestamp up to which the backup will stay pinned. Must be a ISO-8601 complaint timestamp. Example: `--expire-time='2020-01-01 00:00:00+03'` - -#### Logging Options - -You can use these options with any command. - - --log-level-console=log_level - Default: info -Controls which message levels are sent to the console log. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables console logging. - ->NOTE: all console log messages are going to stderr, so output from [show](#show) and [show-config](#show-config) commands do not mingle with log messages. - - --log-level-file=log_level - Default: off -Controls which message levels are sent to a log file. Valid values are `verbose`, `log`, `info`, `warning`, `error` and `off`. Each level includes all the levels that follow it. The later the level, the fewer messages are sent. The `off` level disables file logging. - - --log-filename=log_filename - Default: pg_probackup.log -Defines the filenames of the created log files. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. - -For example, if you specify the 'pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: pg_probackup-1.log for Monday, pg_probackup-2.log for Tuesday, and so on. - -This option takes effect if file logging is enabled by the `log-level-file` option. - - --error-log-filename=error_log_filename - Default: none -Defines the filenames of log files for error messages only. The filenames are treated as a strftime pattern, so you can use %-escapes to specify time-varying filenames. - -For example, if you specify the 'error-pg_probackup-%u.log' pattern, pg_probackup generates a separate log file for each day of the week, with %u replaced by the corresponding decimal number: error-pg_probackup-1.log for Monday, error-pg_probackup-2.log for Tuesday, and so on. - -This option is useful for troubleshooting and monitoring. - - --log-directory=log_directory - Default: $BACKUP_PATH/log/ -Defines the directory in which log files will be created. You must specify the absolute path. This directory is created lazily, when the first log message is written. - - --log-rotation-size=log_rotation_size - Default: 0 -Maximum size of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The zero value disables size-based rotation. Supported units: kB, MB, GB, TB (kB by default). - - --log-rotation-age=log_rotation_age - Default: 0 -Maximum lifetime of an individual log file. If this value is reached, the log file is rotated once a pg_probackup command is launched, except help and version commands. The time of the last log file creation is stored in $BACKUP_PATH/log/log_rotation. The zero value disables time-based rotation. Supported units: ms, s, min, h, d (min by default). - -#### Connection Options - -You can use these options together with [backup](#backup) and [checkdb](#checkdb) commands. - -All [libpq environment variables](https://www.postgresql.org/docs/current/libpq-envars.html) are supported. - - -d dbname - --pgdatabase=dbname - PGDATABASE -Specifies the name of the database to connect to. The connection is used only for managing backup process, so you can connect to any existing database. If this option is not provided on the command line, PGDATABASE environment variable, or the pg_probackup.conf configuration file, pg_probackup tries to take this value from the PGUSER environment variable, or from the current user name if PGUSER variable is not set. - - -h host - --pghost=host - PGHOST - Default: local socket -Specifies the host name of the system on which the server is running. If the value begins with a slash, it is used as a directory for the Unix domain socket. - - -p port - --pgport=port - PGPORT - Default: 5432 -Specifies the TCP port or the local Unix domain socket file extension on which the server is listening for connections. - - -U username - --pguser=username - PGUSER -User name to connect as. - - -w - --no-password - Disables a password prompt. If the server requires password authentication and a password is not available by other means such as a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file or PGPASSWORD environment variable, the connection attempt will fail. This flag can be useful in batch jobs and scripts where no user is present to enter a password. - - -W - --password -Forces a password prompt. - -#### Compression Options - -You can use these options together with [backup](#backup) and [archive-push](#archive-push) commands. - - --compress-algorithm=compression_algorithm - Default: none -Defines the algorithm to use for compressing data files. Possible values are `zlib`, `pglz`, and `none`. If set to zlib or pglz, this option enables compression. By default, compression is disabled. -For the [archive-push](#archive-push) command, the pglz compression algorithm is not supported. - - --compress-level=compression_level - Default: 1 -Defines compression level (0 through 9, 0 being no compression and 9 being best compression). This option can be used together with `--compress-algorithm` option. - - --compress -Alias for `--compress-algorithm=zlib` and `--compress-level=1`. - -#### Archiving Options - -These options can be used with [archive-push](#archive-push) command in [archive_command](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND) setting and [archive-get](#archive-get) command in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) setting. - -Additionally [Remote Mode Options](#remote-mode-options) and [Logging Options](#logging-options) can be used. - - --wal-file-path=wal_file_path -Provides the path to the WAL file in `archive_command` and `restore_command`. The `%p` variable as value for this option is required for correct processing. - - --wal-file-name=wal_file_name -Provides the name of the WAL file in `archive_command` and `restore_command`. The `%f` variable as value is required for correct processing. - - --overwrite -Overwrites archived WAL file. Use this flag together with the [archive-push](#archive-push) command if the specified subdirectory of the backup catalog already contains this WAL file and it needs to be replaced with its newer copy. Otherwise, archive-push reports that a WAL segment already exists, and aborts the operation. If the file to replace has not changed, archive-push skips this file regardless of the `--overwrite` flag. - -#### Remote Mode Options - -This section describes the options related to running pg_probackup operations remotely via SSH. These options can be used with [add-instance](#add-instance), [set-config](#set-config), [backup](#backup), [restore](#restore), [archive-push](#archive-push) and [archive-get](#archive-get) commands. - -For details on configuring and usage of remote operation mode, see the sections [Configuring the Remote Mode](#configuring-the-remote-mode) and [Using pg_probackup in the Remote Mode](#using-pg_probackup-in-the-remote-mode). - - --remote-proto=proto -Specifies the protocol to use for remote operations. Currently only the SSH protocol is supported. Possible values are: - -- `ssh` enables the remote backup mode via SSH. This is the Default value. -- `none` explicitly disables the remote mode. - -You can omit this option if the `--remote-host` option is specified. - - --remote-host=destination -Specifies the remote host IP address or hostname to connect to. - - --remote-port=port - Default: 22 -Specifies the remote host port to connect to. - - --remote-user=username - Default: current user -Specifies remote host user for SSH connection. If you omit this option, the current user initiating the SSH connection is used. - - --remote-path=path -Specifies pg_probackup installation directory on the remote system. - - --ssh-options=ssh_options -Specifies a string of SSH command-line options. For example, the following options can used to set keep-alive for ssh connections opened by pg_probackup: `--ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'`. Full list of possible options can be found on [ssh_config manual page](https://man.openbsd.org/ssh_config.5). - -#### Remote WAL Archive Options - -This section describes the options used to provide the arguments for [Remote Mode Options](#remote-mode-options) in [archive-get](#archive-get) used in [restore_command](https://www.postgresql.org/docs/current/archive-recovery-settings.html#RESTORE-COMMAND) command when restoring ARCHIVE backup or performing PITR. - - --archive-host=destination -Provides the argument for `--remote-host` option in `archive-get` command. - - --archive-port=port - Default: 22 -Provides the argument for `--remote-port` option in `archive-get` command. - - --archive-user=username - Default: PostgreSQL user -Provides the argument for `--remote-user` option in `archive-get` command. If you omit this option, the the user running PostgreSQL cluster is used. - -#### Partial Restore Options - -This section describes the options related to partial restore of a cluster from backup. These options can be used with [restore](#restore) command. - - --db-exclude=dbname -Specifies database name to exclude from restore. All other databases in the cluster will be restored as usual, including `template0` and `template1`. This option can be specified multiple times for multiple databases. - - --db-include=dbname -Specifies database name to restore from backup. All other databases in the cluster will not be restored, with exception of `template0` and `template1`. This option can be specified multiple times for multiple databases. - -#### Replica Options - -This section describes the options related to taking a backup from standby. - ->NOTE: Starting from pg_probackup 2.0.24, backups can be taken from standby without connecting to the master server, so these options are no longer required. In lower versions, pg_probackup had to connect to the master to determine recovery time — the earliest moment for which you can restore a consistent state of the database cluster. - - --master-db=dbname - Default: postgres, the default PostgreSQL database. -Deprecated. Specifies the name of the database on the master server to connect to. The connection is used only for managing the backup process, so you can connect to any existing database. Can be set in the pg_probackup.conf using the [set-config](#set-config) command. - - --master-host=host -Deprecated. Specifies the host name of the system on which the master server is running. - - --master-port=port - Default: 5432, the PostgreSQL default port. -Deprecated. Specifies the TCP port or the local Unix domain socket file extension on which the master server is listening for connections. - - --master-user=username - Default: postgres, the PostgreSQL default user name. -Deprecated. User name to connect as. - - --replica-timeout=timeout - Default: 300 sec -Deprecated. Wait time for WAL segment streaming via replication, in seconds. By default, pg_probackup waits 300 seconds. You can also define this parameter in the pg_probackup.conf configuration file using the [set-config](#set-config) command. - -## Howto - -All examples below assume the remote mode of operations via `ssh`. If you are planning to run backup and restore operation locally then step `Setup passwordless SSH connection` can be skipped and all `--remote-*` options can be ommited. - -Examples are based on Ubuntu 18.04, PostgreSQL 11 and pg_probackup 2.2.0. - -- *backup_host* - host with backup catalog. -- *backupman* - user on `backup_host` running all pg_probackup operations. -- */mnt/backups* - directory on `backup_host` where backup catalog is stored. -- *postgres_host* - host with PostgreSQL cluster. -- *postgres* - user on `postgres_host` which run PostgreSQL cluster. -- */var/lib/postgresql/11/main* - directory on `postgres_host` where PGDATA of PostgreSQL cluster is located. -- *backupdb* - database used for connection to PostgreSQL cluster. - -### Minimal Setup - -This setup is relying on autonomous FULL and DELTA backups. - -#### Setup passwordless SSH connection from `backup_host` to `postgres_host` -``` -[backupman@backup_host] ssh-copy-id postgres@postgres_host -``` - -#### Setup PostgreSQL cluster - -It is recommended from security purposes to use separate database for backup operations. -``` -postgres=# -CREATE DATABASE backupdb; -``` - -Connect to `backupdb` database, create role `probackup` and grant to it the following permissions: -``` -backupdb=# -BEGIN; -CREATE ROLE probackup WITH LOGIN REPLICATION; -GRANT USAGE ON SCHEMA pg_catalog TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO probackup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO probackup; -COMMIT; -``` - -#### Init the backup catalog -``` -[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups -INFO: Backup catalog '/mnt/backups' successfully inited -``` - -#### Add instance 'pg-11' to backup catalog -``` -[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main -INFO: Instance 'node' successfully inited -``` - -#### Take FULL backup -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YK2 -INFO: Backup PZ7YK2 data files are valid -INFO: Backup PZ7YK2 resident size: 196MB -INFO: Backup PZ7YK2 completed -``` - -#### Lets take a look at the backup catalog -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' - -BACKUP INSTANCE 'pg-11' -================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK -``` - -#### Take incremental backup in DELTA mode -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YK2 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YMP -INFO: Backup PZ7YMP data files are valid -INFO: Backup PZ7YMP resident size: 32MB -INFO: Backup PZ7YMP completed -``` - -#### Lets hide some parameters into config, so cmdline can be less crowdy -``` -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U probackup -d backupdb -``` - -#### Take another incremental backup in DELTA mode, omitting some of the previous parameters: -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YMP -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YR5 -INFO: Backup PZ7YR5 data files are valid -INFO: Backup PZ7YR5 resident size: 32MB -INFO: Backup PZ7YR5 completed -``` - -#### Lets take a look at instance config -``` -[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' - -# Backup instance information -pgdata = /var/lib/postgresql/11/main -system-identifier = 6746586934060931492 -xlog-seg-size = 16777216 -# Connection parameters -pgdatabase = backupdb -pghost = postgres_host -pguser = probackup -# Replica parameters -replica-timeout = 5min -# Archive parameters -archive-timeout = 5min -# Logging parameters -log-level-console = INFO -log-level-file = OFF -log-filename = pg_probackup.log -log-rotation-size = 0 -log-rotation-age = 0 -# Retention parameters -retention-redundancy = 0 -retention-window = 0 -wal-depth = 0 -# Compression parameters -compress-algorithm = none -compress-level = 1 -# Remote access parameters -remote-proto = ssh -remote-host = postgres_host -``` - -Note, that we are getting default values for other options, that were not overwritten by set-config command. - - -#### Lets take a look at the backup catalog -``` -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' - -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK -``` - -## Authors -Postgres Professional, Moscow, Russia. - -## Credits -pg_probackup utility is based on pg_arman, that was originally written by NTT and then developed and maintained by Michael Paquier. diff --git a/LICENSE b/LICENSE index dc4e8b8d5..66476e8a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015-2019, Postgres Professional +Copyright (c) 2015-2023, Postgres Professional Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group diff --git a/Makefile b/Makefile index 41502286b..f93cc37a4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ PROGRAM = pg_probackup +WORKDIR ?= $(CURDIR) +BUILDDIR = $(WORKDIR)/build/ +PBK_GIT_REPO = https://github.com/postgrespro/pg_probackup # utils OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ @@ -6,103 +9,81 @@ OBJS = src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ - src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/util.o \ - src/validate.o + src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ + src/util.o src/validate.o src/datapagemap.o src/catchup.o # borrowed files -OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \ +OBJS += src/pg_crc.o src/receivelog.o src/streamutil.o \ src/xlogreader.o -EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h \ +EXTRA_CLEAN = src/pg_crc.c \ src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \ src/xlogreader.c src/instr_time.h -INCLUDES = src/datapagemap.h src/streamutil.h src/receivelog.h src/instr_time.h +ifdef top_srcdir +srchome := $(abspath $(top_srcdir)) +else +top_srcdir=../.. +ifneq (,$(wildcard ../../../contrib/pg_probackup)) +# separate build directory support +srchome := $(abspath $(top_srcdir)/..) +else +srchome := $(abspath $(top_srcdir)) +endif +endif + +# OBJS variable must be finally defined before invoking the include directive +ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) +OBJS += src/walmethods.o +EXTRA_CLEAN += src/walmethods.c src/walmethods.h +endif ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# !USE_PGXS else subdir=contrib/pg_probackup top_builddir=../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk -endif # USE_PGXS - -ifeq ($(top_srcdir),../..) - ifeq ($(LN_S),ln -s) - srchome=$(top_srcdir)/.. - endif -else -srchome=$(top_srcdir) endif -#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) -ifneq (12,$(MAJORVERSION)) -EXTRA_CLEAN += src/logging.h -INCLUDES += src/logging.h -endif - -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -OBJS += src/walmethods.o -EXTRA_CLEAN += src/walmethods.c src/walmethods.h -INCLUDES += src/walmethods.h -endif - - -PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(top_srcdir)/$(subdir)/src +PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc -I$(srchome)/$(subdir)/src override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} -all: checksrcdir $(INCLUDES); - -$(PROGRAM): $(OBJS) +src/utils/configuration.o: src/datapagemap.h +src/archive.o: src/instr_time.h +src/backup.o: src/receivelog.h src/streamutil.h -src/instr_time.h: $(top_srcdir)/src/include/portability/instr_time.h +src/instr_time.h: $(srchome)/src/include/portability/instr_time.h rm -f $@ && $(LN_S) $(srchome)/src/include/portability/instr_time.h $@ -src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@ -src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@ -src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c +src/pg_crc.c: $(srchome)/src/backend/utils/hash/pg_crc.c rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@ -src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c +src/receivelog.c: $(srchome)/src/bin/pg_basebackup/receivelog.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@ -src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h +ifneq (,$(wildcard $(srchome)/src/bin/pg_basebackup/walmethods.c)) +src/receivelog.h: src/walmethods.h $(srchome)/src/bin/pg_basebackup/receivelog.h +else +src/receivelog.h: $(srchome)/src/bin/pg_basebackup/receivelog.h +endif rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@ -src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c +src/streamutil.c: $(srchome)/src/bin/pg_basebackup/streamutil.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@ -src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h +src/streamutil.h: $(srchome)/src/bin/pg_basebackup/streamutil.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@ -src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c +src/xlogreader.c: $(srchome)/src/backend/access/transam/xlogreader.c rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@ - -#ifneq (,$(filter 9.5 9.6 10 11,$(MAJORVERSION))) -ifneq (12,$(MAJORVERSION)) -src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h - rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@ -endif - -ifeq (,$(filter 9.5 9.6,$(MAJORVERSION))) -src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c +src/walmethods.c: $(srchome)/src/bin/pg_basebackup/walmethods.c rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@ -src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h +src/walmethods.h: $(srchome)/src/bin/pg_basebackup/walmethods.h rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@ -endif ifeq ($(PORTNAME), aix) CC=xlc_r endif -# This rule's only purpose is to give the user instructions on how to pass -# the path to PostgreSQL source tree to the makefile. -.PHONY: checksrcdir -checksrcdir: -ifndef top_srcdir - @echo "You must have PostgreSQL source tree available to compile." - @echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir" - @echo "variable: \"make top_srcdir=\"" - @exit 1 -endif +include packaging/Makefile.pkg +include packaging/Makefile.repo +include packaging/Makefile.test diff --git a/README.md b/README.md index 813abbcc8..2279b97a4 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ -[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) +[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) # pg_probackup `pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. The utility is compatible with: -* PostgreSQL 9.5, 9.6, 10, 11, 12; +* PostgreSQL 11, 12, 13, 14, 15, 16 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data: * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow. +* Incremental restore: page-level incremental restore allows you dramatically speed up restore by reusing valid unchanged pages in destination directory. * Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to do periodical full backups. * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery * Verification: on-demand verification of PostgreSQL instance with the `checkdb` command. @@ -40,10 +41,9 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp ## ptrack support `PTRACK` backup support provided via following options: -* vanilla PostgreSQL compiled with ptrack patch. Currently there are patches for [PostgreSQL 9.6](https://gist.githubusercontent.com/gsmol/5b615c971dfd461c76ef41a118ff4d97/raw/e471251983f14e980041f43bea7709b8246f4178/ptrack_9.6.6_v1.5.patch) and [PostgreSQL 10](https://gist.githubusercontent.com/gsmol/be8ee2a132b88463821021fd910d960e/raw/de24f9499f4f314a4a3e5fae5ed4edb945964df8/ptrack_10.1_v1.5.patch) -* vanilla PostgreSQL 12 with [ptrack extension](https://github.com/postgrespro/ptrack) -* Postgres Pro Standard 9.6, 10, 11, 12 -* Postgres Pro Enterprise 9.6, 10, 11, 12 +* vanilla PostgreSQL 11, 12, 13, 14, 15, 16 with [ptrack extension](https://github.com/postgrespro/ptrack) +* Postgres Pro Standard 11, 12, 13, 14, 15, 16 +* Postgres Pro Enterprise 11, 12, 13, 14, 15, 16 ## Limitations @@ -52,111 +52,29 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp * Remote backup via ssh on Windows currently is not supported. * When running remote operations via ssh, remote and local pg_probackup versions must be the same. -## Current release - -[2.3.5](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.5) - ## Documentation Documentation can be found at [github](https://postgrespro.github.io/pg_probackup) and [Postgres Professional documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup) +## Development + +* Stable version state can be found under the respective [release tag](https://github.com/postgrespro/pg_probackup/releases). +* `master` branch contains minor fixes that are planned to the nearest minor release. +* Upcoming major release is developed in a release branch i.e. `release_2_6`. + +For detailed release plans check [Milestones](https://github.com/postgrespro/pg_probackup/milestones) + ## Installation and Setup ### Windows Installation -Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/2.3.5). +Installers are available in release **assets**. [Latests](https://github.com/postgrespro/pg_probackup/releases/latest). ### Linux Installation -#### pg_probackup for vanilla PostgreSQL -```shell -#DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup.list -sudo wget -O - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg-probackup-{12,11,10,9.6,9.5}-dbg - -#DEB-SRC Packages -sudo echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ - /etc/apt/sources.list.d/pg_probackup.list && sudo apt-get update -sudo apt-get source pg-probackup-{12,11,10,9.6,9.5} - -#RPM Centos Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo - -#RPM RHEL Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-rhel.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo - -#RPM Oracle Linux Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-oraclelinux.noarch.rpm -yum install pg_probackup-{12,11,10,9.6,9.5} -yum install pg_probackup-{12,11,10,9.6,9.5}-debuginfo - -#SRPM Packages -yumdownloader --source pg_probackup-{12,11,10,9.6,9.5} - -#RPM ALT Linux 7 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list -sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo - -#RPM ALT Linux 8 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p8 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list -sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo - -#RPM ALT Linux 9 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p9 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list -sudo apt-get update -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5} -sudo apt-get install pg_probackup-{12,11,10,9.6,9.5}-debuginfo -``` -#### pg_probackup for PostgresPro Standard and Enterprise -```shell -#DEB Ubuntu|Debian Packages -sudo echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" > /etc/apt/sources.list.d/pg_probackup-forks.list -sudo wget -O - https://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | sudo apt-key add - && sudo apt-get update -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg-probackup-{std,ent}-{12,11,10,9.6}-dbg - -#RPM Centos Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-centos.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo - -#RPM RHEL Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-rhel.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo - -#RPM Oracle Linux Packages -rpm -ivh https://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-oraclelinux.noarch.rpm -yum install pg_probackup-{std,ent}-{12,11,10,9.6} -yum install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo - -#RPM ALT Linux 7 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p7 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo - -#RPM ALT Linux 8 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p8 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list -sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo - -#RPM ALT Linux 9 -sudo echo "rpm https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p9 x86_64 forks" > /etc/apt/sources.list.d/pg_probackup_forks.list && sudo apt-get update -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6} -sudo apt-get install pg_probackup-{std,ent}-{12,11,10,9.6}-debuginfo -``` +See the [Installation](https://postgrespro.github.io/pg_probackup/#pbk-install) section in the documentation. + +Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.github.io/pg_probackup/#pbk-setup). -Once you have `pg_probackup` installed, complete [the setup](https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#installation-and-setup). +For users of Postgres Pro products, commercial editions of pg_probackup are available for installation from the corresponding Postgres Pro product repository. ## Building from source ### Linux @@ -176,7 +94,7 @@ cd && git clone https://github.com/postgrespro/ ### Windows Currently pg_probackup can be build using only MSVC 2013. -Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/10/install-windows-full.html) with MSVC 2013. +Build PostgreSQL using [pgwininstall](https://github.com/postgrespro/pgwininstall) or [PostgreSQL instruction](https://www.postgresql.org/docs/current/install-windows-full.html) with MSVC 2013. If zlib support is needed, src/tools/msvc/config.pl must contain path to directory with compiled zlib. [Example](https://gist.githubusercontent.com/gsmol/80989f976ce9584824ae3b1bfb00bd87/raw/240032950d4ac4801a79625dd00c8f5d4ed1180c/gistfile1.txt) ```shell @@ -201,3 +119,17 @@ Postgres Professional, Moscow, Russia. ## Credits `pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier. + + +### Localization files (*.po) + +Description of how to add new translation languages. +1. Add a flag --enable-nls in configure. +2. Build postgres. +3. Adding to nls.mk in folder pg_probackup required files in GETTEXT_FILES. +4. In folder pg_probackup do 'make update-po'. +5. As a result, the progname.pot file will be created. Copy the content and add it to the file with the desired language. +6. Adding to nls.mk in folder pg_probackup required language in AVAIL_LANGUAGES. + +For more information, follow the link below: +https://postgrespro.ru/docs/postgresql/12/nls-translator diff --git a/doc/Readme.md b/doc/Readme.md index b9c74769e..0e1d64590 100644 --- a/doc/Readme.md +++ b/doc/Readme.md @@ -2,4 +2,7 @@ ``` xmllint --noout --valid probackup.xml xsltproc stylesheet.xsl probackup.xml >pg-probackup.html -``` \ No newline at end of file +``` +> [!NOTE] +>Install ```docbook-xsl``` if you got +>``` "xsl:import : unable to load http://docbook.sourceforge.net/release/xsl/current/xhtml/docbook.xsl"``` \ No newline at end of file diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml index 26211b30e..10e766239 100644 --- a/doc/pgprobackup.xml +++ b/doc/pgprobackup.xml @@ -2,7 +2,6 @@ doc/src/sgml/pgprobackup.sgml &project; documentation --> - pg_probackup @@ -131,6 +130,7 @@ doc/src/sgml/pgprobackup.sgml backup_dir instance_name + wal_file_path wal_file_name option @@ -143,6 +143,14 @@ doc/src/sgml/pgprobackup.sgml wal_file_name option + + pg_probackup + + catchup_mode + =path_to_pgdata_on_remote_server + =path_to_local_dir + option + @@ -155,7 +163,7 @@ doc/src/sgml/pgprobackup.sgml recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure. - pg_probackup supports PostgreSQL 9.5 or higher. + pg_probackup supports PostgreSQL 11 or higher. @@ -163,7 +171,13 @@ doc/src/sgml/pgprobackup.sgml Overview - Installation and Setup + Quick Start + + + Installation + + + Setup Command-Line Reference @@ -283,6 +297,11 @@ doc/src/sgml/pgprobackup.sgml Partial restore: restoring only the specified databases. + + + Catchup: cloning a PostgreSQL instance for a fallen-behind standby server to catch up with master. + + To manage backup data, pg_probackup creates a @@ -298,7 +317,7 @@ doc/src/sgml/pgprobackup.sgml - + FULL backups contain all the data files required to restore the database cluster. @@ -314,7 +333,7 @@ doc/src/sgml/pgprobackup.sgml - + DELTA backup. In this mode, pg_probackup reads all data files in the data directory and copies only those pages that have changed since the previous backup. This @@ -323,7 +342,7 @@ doc/src/sgml/pgprobackup.sgml - + PAGE backup. In this mode, pg_probackup scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups @@ -338,7 +357,7 @@ doc/src/sgml/pgprobackup.sgml - + PTRACK backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, @@ -389,7 +408,7 @@ doc/src/sgml/pgprobackup.sgml - pg_probackup only supports PostgreSQL 9.5 and higher. + pg_probackup only supports PostgreSQL 9.5 and higher. @@ -399,7 +418,7 @@ doc/src/sgml/pgprobackup.sgml - On Unix systems, for PostgreSQL 10 or higher, + On Unix systems, for PostgreSQL 11, a backup can be made only by the same OS user that has started the PostgreSQL server. For example, if PostgreSQL server is started by user postgres, the backup command must also be run @@ -410,7 +429,7 @@ doc/src/sgml/pgprobackup.sgml - For PostgreSQL 9.5, functions + For PostgreSQL 9.5, functions pg_create_restore_point(text) and pg_switch_xlog() can be executed only if the backup role is a superuser, so backup of a @@ -429,16 +448,575 @@ doc/src/sgml/pgprobackup.sgml parameters and have the same major release number. Depending on cluster configuration, PostgreSQL itself may apply additional restrictions, such as CPU architecture - or libc/libicu versions. + or libc/icu versions. - - - Installation and Setup + + Quick Start + + To quickly get started with pg_probackup, complete the steps below. This will set up FULL and DELTA backups in the remote mode and demonstrate some + basic pg_probackup operations. In the following, these terms are used: + + + + + backup — PostgreSQL + role used to connect to the PostgreSQL + cluster. + + + + + backupdb — database used to connect to the + PostgreSQL cluster. + + + + + backup_host — host with the backup catalog. + + + + + backup_user — user on + backup_host running all pg_probackup + operations. + + + + + /mnt/backups — directory on + backup_host where the backup catalog is stored. + + + + + postgres_host — host with the + PostgreSQL cluster. + + + + + postgres — user on + postgres_host under which + PostgreSQL cluster processes are running. + + + + + /var/lib/postgresql/16/main — + PostgreSQL data directory on + postgres_host. + + + + + Steps to perform: + + + Install pg_probackup on both backup_host and postgres_host. + + + Set up an SSH connection from backup_host to postgres_host. + + + Configure your database cluster for STREAM backups. + + + Initialize the backup catalog: + +backup_user@backup_host:~$ pg_probackup-16 init -B /mnt/backups +INFO: Backup catalog '/mnt/backups' successfully initialized + + + + Add a backup instance called mydb to the backup catalog: + +backup_user@backup_host:~$ pg_probackup-16 add-instance \ + -B /mnt/backups \ + -D /var/lib/pgpro/std-16/data \ + --instance=node \ + --remote-host=postgres_host \ + --remote-user=postgres +INFO: Instance 'node' successfully initialized + + + + Make a FULL backup: + +backup_user@backup_host:~$ pg_probackup-16 backup \ + -B /mnt/backups \ + -b FULL \ + --instance=node \ + --stream \ + --compress-algorithm=zlib \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN1Q, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN1Q/database/pg_wal/000000010000000000000008 to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/8000028, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 1s +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/800BBD0 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 1s +INFO: Validating backup SCUN1Q +INFO: Backup SCUN1Q data files are valid +INFO: Backup SCUN1Q resident size: 56MB +INFO: Backup SCUN1Q completed + + + + List the backups of the instance: + +backup_user@backup_host:~$ pg_probackup-16 show \ + -B /mnt/backups \ + --instance=node +================================================================================================================================ + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +================================================================================================================================ + node 16 SCUN1Q 2024-05-02 11:17:53+03 FULL STREAM 1/0 12s 40MB 16MB 2.42 0/8000028 0/800BBD0 OK + + + + Make an incremental backup in the DELTA mode: + +backup_user@backup_host:~$ pg_probackup-16 backup \ + -B /mnt/backups \ + -b DELTA \ + --instance=node \ + --stream \ + --compress-algorithm=zlib \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN22, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Parent backup: SCUN1Q +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN22/database/pg_wal/000000010000000000000009 to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/9000028, TLI: 1 +INFO: Parent Start LSN: 0/8000028, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 1s +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/9000168 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 1s +INFO: Validating backup SCUN22 +INFO: Backup SCUN22 data files are valid +INFO: Backup SCUN22 resident size: 34MB +INFO: Backup SCUN22 completed + + + + Add or modify some parameters in the pg_probackup + configuration file, so that you do not have to specify them each time on the command line: + +backup_user@backup_host:~$ pg_probackup-16 set-config \ + -B /mnt/backups \ + --instance=node \ + --remote-host=postgres_host \ + --remote-user=postgres \ + -U backup \ + -d backupdb + + + + Check the configuration of the instance: + +backup_user@backup_host:~$ pg_probackup-16 show-config \ + -B /mnt/backups \ + --instance=node +# Backup instance information +pgdata = /var/lib/pgpro/std-16/data +system-identifier = 7364313570668255886 +xlog-seg-size = 16777216 +# Connection parameters +pgdatabase = backupdb +pghost = postgres_host +pguser = backup +# Replica parameters +replica-timeout = 5min +# Archive parameters +archive-timeout = 5min +# Logging parameters +log-level-console = INFO +log-level-file = OFF +log-format-console = PLAIN +log-format-file = PLAIN +log-filename = pg_probackup.log +log-rotation-size = 0TB +log-rotation-age = 0d +# Retention parameters +retention-redundancy = 0 +retention-window = 0 +wal-depth = 0 +# Compression parameters +compress-algorithm = none +compress-level = 1 +# Remote access parameters +remote-proto = ssh +remote-host = postgres_host +remote-user = postgres + + + Note that the parameters not modified via set-config retain their default values. + + + + Make another incremental backup in the DELTA mode, omitting + the parameters stored in the configuration file earlier: + +backup_user@backup_host:~$ pg_probackup-16 backup \ + -B /mnt/backups \ + -b DELTA \ + --instance=node \ + --stream \ + --compress-algorithm=zlib +INFO: Backup start, pg_probackup version: 2.5.15, instance: node, backup ID: SCUN2C, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: zlib, compress-level: 1 +INFO: This PostgreSQL instance was initialized with data block checksums. Data block corruption will be detected +INFO: Database backup start +INFO: wait for pg_backup_start() +INFO: Parent backup: SCUN22 +INFO: Wait for WAL segment /mnt/backups/backups/node/SCUN2C/database/pg_wal/00000001000000000000000B to be streamed +INFO: PGDATA size: 96MB +INFO: Current Start LSN: 0/B000028, TLI: 1 +INFO: Parent Start LSN: 0/9000028, TLI: 1 +INFO: Start transferring data files +INFO: Data files are transferred, time elapsed: 0 +INFO: wait for pg_stop_backup() +INFO: pg_stop backup() successfully executed +INFO: stop_lsn: 0/B000168 +INFO: Getting the Recovery Time from WAL +INFO: Syncing backup files to disk +INFO: Backup files are synced, time elapsed: 0 +INFO: Validating backup SCUN2C +INFO: Backup SCUN2C data files are valid +INFO: Backup SCUN2C resident size: 17MB +INFO: Backup SCUN2C completed + + + + List the backups of the instance again: + +backup_user@backup_host:~$ pg_probackup-16 show \ + -B /mnt/backups \ + --instance=node +=================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +=================================================================================================================================== + node 16 SCUN2C 2024-05-02 11:18:13+03 DELTA STREAM 1/1 10s 1139kB 16MB 1.00 0/B000028 0/B000168 OK + node 16 SCUN22 2024-05-02 11:18:04+03 DELTA STREAM 1/1 10s 2357kB 32MB 1.02 0/9000028 0/9000168 OK + node 16 SCUN1Q 2024-05-02 11:17:53+03 FULL STREAM 1/0 12s 40MB 16MB 2.42 0/8000028 0/800BBD0 OK + + + + Restore the data from the latest available backup to an arbitrary location: + +backup_user@backup_host:~$ pg_probackup-16 restore \ + -B /mnt/backups \ + -D /var/lib/pgpro/std-16/staging-data \ + --instance=node +INFO: Validating parents for backup SCUN2C +INFO: Validating backup SCUN1Q +INFO: Backup SCUN1Q data files are valid +INFO: Validating backup SCUN22 +INFO: Backup SCUN22 data files are valid +INFO: Validating backup SCUN2C +INFO: Backup SCUN2C data files are valid +INFO: Backup SCUN2C WAL segments are valid +INFO: Backup SCUN2C is valid. +INFO: Restoring the database from backup SCUN2C on localhost +INFO: Start restoring backup files. PGDATA size: 112MB +INFO: Backup files are restored. Transfered bytes: 112MB, time elapsed: 0 +INFO: Restore incremental ratio (less is better): 100% (112MB/112MB) +INFO: Syncing restored files to disk +INFO: Restored backup files are synced, time elapsed: 2s +INFO: Restore of backup SCUN2C completed. + + + + + + + Installation + + Installation on Debian family systems (Debian, Ubuntu etc.) + + You may need to use apt-get instead of apt on older systems in the commands below. + + + + + Add the pg_probackup repository GPG key + + +sudo apt install gpg wget +wget -qO - https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG-PROBACKUP | \ +sudo tee /etc/apt/trusted.gpg.d/pg_probackup.asc + + + + + Setup the binary package repository + + +. /etc/os-release +echo "deb [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + + Optionally setup the source package repository for rebuilding the binaries + + +echo "deb-src [arch=amd64] https://repo.postgrespro.ru/pg_probackup/deb $VERSION_CODENAME main-$VERSION_CODENAME" | \ +sudo tee -a /etc/apt/sources.list.d/pg_probackup.list + + + + + List the available pg_probackup packages + + + + + Using apt: + + +sudo apt update +apt search pg_probackup + + + + + Using apt-get: + + +sudo apt-get update +apt-cache search pg_probackup + + + + + + + Install or upgrade a pg_probackup version of your choice + + +sudo apt install pg-probackup-16 + + + + + Optionally install the debug package + + +sudo apt install pg-probackup-16-dbg + + + + + Optionally install the source package (provided you have set up the source package repository as described above) + + +sudo apt install dpkg-dev +sudo apt source pg-probackup-16 + + + + + + Installation on Red Hat family systems (CentOS, Oracle Linux etc.) + + You may need to use yum instead of dnf on older systems in the commands below. + + + + + Install the pg_probackup repository + + +dnf install https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-centos.noarch.rpm + + + + + List the available pg_probackup packages + + +dnf search pg_probackup + + + + + Install or upgrade a pg_probackup version of your choice + + +dnf install pg_probackup-16 + + + + + Optionally install the debug package + + +dnf install pg_probackup-16-debuginfo + + + + + Optionally install the source package for rebuilding the binaries + + + + + Using dnf: + + +dnf install 'dnf-command(download)' +dnf download --source pg_probackup-16 + + + + + Using yum: + + +yumdownloader --source pg_probackup-16 + + + + + + + + Installation on ALT Linux + + + + Setup the repository + + + + + On ALT Linux 10: + + +. /etc/os-release +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p$VERSION_ID x86_64 vanilla" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + + On ALT Linux 8 and 9: + + +. /etc/os-release +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-$VERSION_ID x86_64 vanilla" | \ +sudo tee /etc/apt/sources.list.d/pg_probackup.list + + + + + + + List the available pg_probackup packages + + +sudo apt-get update +apt-cache search pg_probackup + + + + + Install or upgrade a pg_probackup version of your choice + + +sudo apt-get install pg_probackup-16 + + + + + Optionally install the debug package + + +sudo apt-get install pg_probackup-16-debuginfo + + + + + + Installation on SUSE Linux + + + + Add the pg_probackup repository GPG key + + +zypper in -y gpg wget +wget -O GPG-KEY-PG_PROBACKUP https://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP +rpm --import GPG-KEY-PG_PROBACKUP + + + + + Setup the repository + + +zypper in https://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-suse.noarch.rpm + + + + + List the available pg_probackup packages + + +zypper se pg_probackup + + + + + Install or upgrade a pg_probackup version of your choice + + +zypper in pg_probackup-16 + + + + + Optionally install the source package for rebuilding the binaries + + +zypper si pg_probackup-16 + + + + + + + Setup Once you have pg_probackup installed, complete the following setup: @@ -520,10 +1098,10 @@ pg_probackup init -B backup_dir To add a new backup instance, run the following command: -pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [remote_options] +pg_probackup add-instance -B backup_dir -D data_dir --instance=instance_name [remote_options] - where: + Where: @@ -555,10 +1133,9 @@ pg_probackup add-instance -B backup_dir -D backups/instance_name directory contains the pg_probackup.conf configuration file that controls - pg_probackup settings for this backup instance. If you run this - command with the - remote_options, the specified - parameters will be added to pg_probackup.conf. + pg_probackup settings for this backup instance. To add + remote_options to the configuration file, use the + command. For details on how to fine-tune pg_probackup configuration, see @@ -592,44 +1169,35 @@ pg_probackup add-instance -B backup_dir -D backup role is used as an example. + + For security reasons, it is recommended to run the configuration SQL queries below + in a separate database. + + +postgres=# CREATE DATABASE backupdb; +postgres=# \c backupdb + To perform a , the following permissions for role backup are required only in the database used for - connection to the PostgreSQL server: - - - For PostgreSQL 9.5: + connection to the PostgreSQL server. - -BEGIN; -CREATE ROLE backup WITH LOGIN; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -COMMIT; - - For PostgreSQL 9.6: + For PostgreSQL versions 11 — 14: BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; @@ -637,16 +1205,17 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; COMMIT; - For PostgreSQL 10 or higher: + For PostgreSQL 15 or higher: BEGIN; CREATE ROLE backup WITH LOGIN; GRANT USAGE ON SCHEMA pg_catalog TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; +GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; @@ -704,7 +1273,18 @@ COMMIT; - Grant the REPLICATION privilege to the backup role: + If the backup role does not exist, create it with + the REPLICATION privilege when + Configuring the + Database Cluster: + + +CREATE ROLE backup WITH LOGIN REPLICATION; + + + + + If the backup role already exists, grant it with the REPLICATION privilege: ALTER ROLE backup WITH REPLICATION; @@ -761,10 +1341,9 @@ ALTER ROLE backup WITH REPLICATION; Setting up Continuous WAL Archiving Making backups in PAGE backup mode, performing - PITR, + PITR and making backups with - ARCHIVE WAL delivery mode and - running incremental backup after timeline switch + ARCHIVE WAL delivery mode require continuous WAL archiving to be enabled. To set up continuous @@ -794,7 +1373,7 @@ ALTER ROLE backup WITH REPLICATION; parameter, as follows: -archive_command = 'install_dir/pg_probackup archive-push -B backup_dir --instance instance_name --wal-file-name=%f [remote_options]' +archive_command = '"install_dir/pg_probackup" archive-push -B "backup_dir" --instance=instance_name --wal-file-name=%f [remote_options]' @@ -864,7 +1443,7 @@ archive_command = 'install_dir/pg_probackup archive-p Setting up Backup from Standby - For PostgreSQL 9.6 or higher, pg_probackup can take backups from + pg_probackup can take backups from a standby server. This requires the following additional setup: @@ -961,6 +1540,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup; +GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup; @@ -986,12 +1566,12 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Configuring the Remote Mode - pg_probackup supports the remote mode that allows to perform - backup, restore and WAL archiving operations remotely. In this - mode, the backup catalog is stored on a local system, while - PostgreSQL instance to backup and/or to restore is located on a - remote system. Currently the only supported remote protocol is - SSH. + pg_probackup supports the remote mode that + allows you to perform backup, restore and WAL archiving operations remotely. + In this mode, the backup catalog is stored on a local system, while + PostgreSQL instance to backup and/or to restore + is located on a remote system. Currently the only supported remote + protocol is SSH. Set up SSH @@ -999,73 +1579,84 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; If you are going to use pg_probackup in remote mode via SSH, complete the following steps: - + Install pg_probackup on both systems: backup_host and - db_host. + postgres_host. - For communication between the hosts set up the passwordless - SSH connection between backup user on - backup_host and + For communication between the hosts set up a passwordless + SSH connection between the backup_user user on + backup_host and the postgres user on - db_host: + postgres_host: -[backup@backup_host] ssh-copy-id postgres@db_host +[backup_user@backup_host] ssh-copy-id postgres@postgres_host + + Where: + + + + + backup_host is the system with + backup catalog. + + + + + postgres_host is the system with the PostgreSQL + cluster. + + + + + backup_user is the OS user on + backup_host used to run pg_probackup. + + + + + postgres is the user on + postgres_host under which + PostgreSQL cluster processes are running. + For PostgreSQL 11 or higher a + more secure approach can be used thanks to + allow-group-access feature. + + + If you are going to rely on continuous - WAL archiving, set up passwordless SSH - connection between postgres user on - db_host and backup + WAL archiving, set up a passwordless SSH + connection between the postgres user on + postgres_host and the backup user on backup_host: -[postgres@db_host] ssh-copy-id backup@backup_host +[postgres@postgres_host] ssh-copy-id backup_user@backup_host - - - where: - - - - - backup_host is the system with - backup catalog. - - - db_host is the system with PostgreSQL - cluster. + Make sure pg_probackup on postgres_host + can be located when a connection via SSH is made. For example, for Bash, you can + modify PATH in ~/.bashrc of the postgres user + (above the line in bashrc that exits the script for non-interactive shells). + Alternatively, for pg_probackup commands, specify the path to the directory + containing the pg_probackup binary on postgres_host via + the --remote-path option. - - - backup is the OS user on - backup_host used to run pg_probackup. - - - - - postgres is the OS user on - db_host used to start the PostgreSQL - cluster. For PostgreSQL 11 or higher a - more secure approach can be used thanks to - allow-group-access - feature. - - - + pg_probackup in the remote mode via SSH works as follows: @@ -1077,7 +1668,8 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; mode: , , , - , + , + , and . @@ -1114,10 +1706,10 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; The main process is usually started on backup_host and connects to - db_host, but in case of + postgres_host, but in case of archive-push and archive-get commands the main process - is started on db_host and connects to + is started on postgres_host and connects to backup_host. @@ -1138,7 +1730,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; Compression is always done on - db_host, while decompression is always done on + postgres_host, while decompression is always done on backup_host. @@ -1161,6 +1753,14 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; PostgreSQL. Links to PTRACK patches can be found here. + + + PTRACK versions lower than 2.0 are deprecated and not supported. Postgres Pro Standard and Postgres Pro Enterprise + versions starting with 11.9.1 contain PTRACK 2.0. Upgrade your server to avoid issues in backups + that you will take in future and be sure to take fresh backups of your clusters with the upgraded + PTRACK since the backups taken with PTRACK 1.x might be corrupt. + + If you are going to use PTRACK backups, complete the following additional steps. The role that will perform PTRACK backups @@ -1168,7 +1768,7 @@ GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; access to all the databases of the cluster. - For PostgreSQL 12 or higher: + For PostgreSQL 11 or higher: @@ -1194,22 +1794,11 @@ CREATE EXTENSION ptrack; together, which leads to false-positive results when tracking changed blocks and increases the incremental backup size as unchanged blocks can also be copied into the incremental backup. - Setting ptrack.map_size to a higher value - does not affect PTRACK operation. The maximum allowed value is 1024. + Setting ptrack.map_size to a higher value does not + affect PTRACK operation, but it is not recommended to set this parameter + to a value higher than 1024. - - - Grant the right to execute PTRACK - functions to the backup role - in the database used to connect to the cluster: - - -GRANT EXECUTE ON FUNCTION pg_ptrack_get_pagemapset(pg_lsn) TO backup; -GRANT EXECUTE ON FUNCTION pg_ptrack_control_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - - @@ -1222,35 +1811,6 @@ GRANT EXECUTE ON FUNCTION pg_ptrack_get_block(oid, oid, oid, bigint) TO backup; - - For older PostgreSQL versions, - PTRACK required taking backups in the exclusive mode - to provide exclusive access to bitmaps with changed blocks. - To set up PTRACK backups for PostgreSQL 11 - or lower, do the following: - - - - - Set the ptrack_enable parameter to - on. - - - - - Grant the right to execute PTRACK - functions to the backup role - in every database of the - cluster: - - -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_clear() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_ptrack_get_and_clear(oid, oid) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; - - - - @@ -1262,41 +1822,16 @@ GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; To create a backup, run the following command: -pg_probackup backup -B backup_dir --instance instance_name -b backup_mode +pg_probackup backup -B backup_dir --instance=instance_name -b backup_mode Where backup_mode can take one of the following values: + FULL, + DELTA, + PAGE, and + PTRACK. - - - - FULL — creates a full backup that contains all the data - files of the cluster to be restored. - - - - - DELTA — reads all data files in the data directory and - creates an incremental backup for pages that have changed - since the previous backup. - - - - - PAGE — creates an incremental backup based on the WAL - files that have been generated since the previous full or - incremental backup was taken. Only changed blocks are read - from data files. - - - - - PTRACK — creates an incremental backup tracking page - changes on the fly. - - - When restoring a cluster from an incremental backup, pg_probackup relies on the parent full backup and all the @@ -1313,7 +1848,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL +pg_probackup backup -B backup_dir --instance=instance_name -b FULL ARCHIVE backups rely on @@ -1343,7 +1878,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --stream --temp-slot +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --stream --temp-slot The optional flag ensures that @@ -1436,7 +1971,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --external-dirs=/etc/dir1:/etc/dir2 Similarly, to include C:\dir1 and @@ -1444,7 +1979,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --external-dirs=C:\dir1;C:\dir2 pg_probackup recursively copies the contents @@ -1464,6 +1999,7 @@ pg_probackup backup -B backup_dir --instance + Performing Cluster Verification @@ -1471,7 +2007,7 @@ pg_probackup backup -B backup_dir --instance -pg_probackup checkdb [-B backup_dir [--instance instance_name]] [-D data_dir] [connection_options] +pg_probackup checkdb [-B backup_dir [--instance=instance_name]] [-D data_dir] [connection_options] @@ -1539,6 +2075,7 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ higher cost of CPU, memory, and I/O consumption. + Validating a Backup @@ -1568,7 +2105,7 @@ pg_probackup checkdb --amcheck --skip-block-validation [connection_ this command: -pg_probackup validate -B backup_dir --instance instance_name --recovery-target-xid=4242 +pg_probackup validate -B backup_dir --instance=instance_name --recovery-target-xid=4242 If validation completes successfully, pg_probackup displays the @@ -1588,11 +2125,11 @@ pg_probackup validate -B backup_dir --instance For example, to check that you can restore the database cluster - from a backup copy with the PT8XFX backup ID up to the + from a backup copy with the SCUN2C backup ID up to the specified timestamp, run this command: - -pg_probackup validate -B backup_dir --instance instance_name -i PT8XFX --recovery-target-time='2017-05-18 14:18:11+03' + +pg_probackup validate -B backup_dir --instance=instance_name -i SCUN2C --recovery-target-time="2024-05-03 11:18:13+03" If you specify the backup_id of an incremental backup, @@ -1610,10 +2147,10 @@ pg_probackup validate -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name -i backup_id +pg_probackup restore -B backup_dir --instance=instance_name -i backup_id - where: + Where: @@ -1657,7 +2194,7 @@ pg_probackup restore -B backup_dir --instance primary_conninfo parameter; you have to add the password manually or use the --primary-conninfo option, if required. - For PostgreSQL 11 or lower, + For PostgreSQL 11, recovery settings are written into the recovery.conf file. Starting from PostgreSQL 12, pg_probackup writes these settings into @@ -1694,7 +2231,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir +pg_probackup restore -B backup_dir --instance=instance_name -D data_dir -j 4 -i backup_id -T tablespace1_dir=tablespace1_newdir -T tablespace2_dir=tablespace2_newdir @@ -1716,7 +2253,7 @@ pg_probackup restore -B backup_dir --instance The speed of restore from backup can be significantly improved by replacing only invalid and changed pages in already - existing PostgreSQL data directory using + existing PostgreSQL data directory using incremental restore options with the command. @@ -1726,7 +2263,7 @@ pg_probackup restore -B backup_dir --instance command with the following options: -pg_probackup restore -B backup_dir --instance instance_name -D data_dir -I incremental_mode +pg_probackup restore -B backup_dir --instance=instance_name -D data_dir -I incremental_mode Where incremental_mode can take one of the @@ -1745,7 +2282,7 @@ pg_probackup restore -B backup_dir --instance LSN — read the pg_control in the - data directory to obtain redo LSN and redo TLI, which allows + data directory to obtain redo LSN and redo TLI, which allows you to determine a point in history(shiftpoint), where data directory state shifted from target backup chain history. If shiftpoint is not within reach of backup chain history, then restore is aborted. @@ -1786,29 +2323,36 @@ pg_probackup restore -B backup_dir --instance - -============================================================================================================================================= - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -============================================================================================================================================= - node 12 QBRNBP 2020-06-11 17:40:58+03 DELTA ARCHIVE 16/15 40s 194MB 16MB 8.26 15/2C000028 15/2D000128 OK - node 12 QBRIDX 2020-06-11 15:51:42+03 PAGE ARCHIVE 15/15 11s 18MB 16MB 5.10 14/DC000028 14/DD0000B8 OK - node 12 QBRIAJ 2020-06-11 15:51:08+03 PAGE ARCHIVE 15/15 20s 141MB 96MB 6.22 14/D4BABFE0 14/DA9871D0 OK - node 12 QBRHT8 2020-06-11 15:45:56+03 FULL ARCHIVE 15/0 2m:11s 1371MB 416MB 10.93 14/9D000028 14/B782E9A0 OK - -pg_probackup restore -B /backup --instance node -R -I lsn -INFO: Running incremental restore into nonempty directory: "/var/lib/pgsql/12/data" -INFO: Destination directory redo point 15/2E000028 on tli 16 is within reach of backup QBRIDX with Stop LSN 14/DD0000B8 on tli 15 -INFO: shift LSN: 14/DD0000B8 -INFO: Restoring the database from backup at 2020-06-11 17:40:58+03 -INFO: Extracting the content of destination directory for incremental restore -INFO: Destination directory content extracted, time elapsed: 1s -INFO: Removing redundant files in destination directory -INFO: Redundant files are removed, time elapsed: 1s -INFO: Start restoring backup files. PGDATA size: 15GB -INFO: Backup files are restored. Transfered bytes: 1693MB, time elapsed: 43s -INFO: Restore incremental ratio (less is better): 11% (1693MB/15GB) -INFO: Restore of backup QBRNBP completed. - + +====================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 16 SCUN3Y 2024-05-02 11:19:16+03 DELTA STREAM 16/15 7s 92MB 208MB 2.27 0/3C0043A8 0/46159C70 OK + node 16 SCUN3M 2024-05-02 11:19:01+03 PTRACK STREAM 15/15 10s 30MB 16MB 2.23 0/32000028 0/32005ED0 OK + node 16 SCUN39 2024-05-02 11:18:50+03 PAGE STREAM 15/15 12s 46MB 32MB 1.44 0/2A000028 0/2B0000B8 OK + node 16 SCUN2V 2024-05-02 11:18:38+03 FULL STREAM 15/0 11s 154MB 16MB 2.32 0/23000028 0/23000168 OK + +backup_user@backup_host:~$ pg_probackup-16 restore -B /mnt/backups --instance=node -R -I lsn +INFO: Destination directory and tablespace directories are empty, disable incremental restore +INFO: Validating parents for backup SCUN3Y +INFO: Validating backup SCUN2V +INFO: Backup SCUN2V data files are valid +INFO: Validating backup SCUN39 +INFO: Backup SCUN39 data files are valid +INFO: Validating backup SCUN3M +INFO: Backup SCUN3M data files are valid +INFO: Validating backup SCUN3Y +INFO: Backup SCUN3Y data files are valid +INFO: Backup SCUN3Y WAL segments are valid +INFO: Backup SCUN3Y is valid. +INFO: Restoring the database from backup SCUN3Y +INFO: Start restoring backup files. PGDATA size: 759MB +INFO: Backup files are restored. Transfered bytes: 759MB, time elapsed: 3s +INFO: Restore incremental ratio (less is better): 100% (759MB/759MB) +INFO: Syncing restored files to disk +INFO: Restored backup files are synced, time elapsed: 1s +INFO: Restore of backup SCUN3Y completed. + Incremental restore is possible only for backups with @@ -1832,7 +2376,7 @@ INFO: Restore of backup QBRNBP completed. with the following options: -pg_probackup restore -B backup_dir --instance instance_name --db-include=database_name +pg_probackup restore -B backup_dir --instance=instance_name --db-include=database_name The option can be specified @@ -1841,14 +2385,14 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --db-include=db1 --db-include=db2 +pg_probackup restore -B backup_dir --instance=instance_name --db-include=db1 --db-include=db2 To exclude one or more databases from restore, use the option: -pg_probackup restore -B backup_dir --instance instance_name --db-exclude=database_name +pg_probackup restore -B backup_dir --instance=instance_name --db-exclude=database_name The option can be specified @@ -1857,7 +2401,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --db-exclude=db1 --db-exclude=db2 +pg_probackup restore -B backup_dir --instance=instance_name --db-exclude=db1 --db-exclude=db2 Partial restore relies on lax behavior of PostgreSQL recovery @@ -1866,12 +2410,27 @@ pg_probackup restore -B backup_dir --instance DROP DATABASE command. + + To decouple a single cluster containing multiple databases into separate clusters with minimal downtime, + you can do partial restore of the cluster as a standby using the option + for specific databases. + The template0 and template1 databases are always restored. + + + Due to recovery specifics of PostgreSQL versions earlier than 12, + it is advisable that you set the + hot_standby + parameter to off when running partial + restore of a PostgreSQL cluster of version earlier than 12. + Otherwise the recovery may fail. + + @@ -1904,7 +2463,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-time='2017-05-18 14:18:11+03' +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-time="2024-05-03 11:18:13+03" @@ -1913,7 +2472,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-xid option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-xid=687 +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-xid=687 @@ -1922,7 +2481,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-lsn option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-lsn=16/B374D848 +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-lsn=16/B374D848 @@ -1931,7 +2490,7 @@ pg_probackup restore -B backup_dir --instance --recovery-target-name option: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target-name='before_app_upgrade' +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target-name="before_app_upgrade" @@ -1941,7 +2500,7 @@ pg_probackup restore -B backup_dir --instance latest value: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target='latest' +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target="latest" @@ -1951,7 +2510,7 @@ pg_probackup restore -B backup_dir --instance immediate value: -pg_probackup restore -B backup_dir --instance instance_name --recovery-target='immediate' +pg_probackup restore -B backup_dir --instance=instance_name --recovery-target='immediate' @@ -1959,7 +2518,7 @@ pg_probackup restore -B backup_dir --instance Using <application>pg_probackup</application> in the Remote Mode - pg_probackup supports the remote mode that allows to perform + pg_probackup supports the remote mode that allows you to perform backup and restore operations remotely via SSH. In this mode, the backup catalog is stored on a local system, while PostgreSQL instance to be backed @@ -1972,6 +2531,15 @@ pg_probackup restore -B backup_dir --instance + + + In addition to SSH connection, pg_probackup uses + a regular connection to the database to manage the remote operation. + See the section Configuring + the Database Cluster for details of how to set up + a database connection. + + The typical workflow is as follows: @@ -1980,8 +2548,7 @@ pg_probackup restore -B backup_dir --instance On your backup host, configure pg_probackup as explained in the section - Installation and - Setup. For the + Setup. For the and commands, make sure to specify remote @@ -2027,7 +2594,7 @@ pg_probackup restore -B backup_dir --instance 2302, run: -pg_probackup backup -B backup_dir --instance instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 To restore the latest available backup on a remote system with host address @@ -2035,7 +2602,7 @@ pg_probackup backup -B backup_dir --instance 2302, run: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 Restoring an ARCHIVE backup or performing PITR in the remote mode @@ -2062,20 +2629,20 @@ pg_probackup restore -B backup_dir --instance 2303, run: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --archive-host=192.168.0.3 --archive-port=2303 --archive-user=backup Provided arguments will be used to construct the restore_command: -restore_command = 'install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +restore_command = '"install_dir/pg_probackup" archive-get -B "backup_dir" --instance=instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' Alternatively, you can use the option to provide the entire restore_command: -pg_probackup restore -B backup_dir --instance instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='install_dir/pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' +pg_probackup restore -B backup_dir --instance=instance_name --remote-user=postgres --remote-host=192.168.0.2 --remote-port=2302 --restore-command='"install_dir/pg_probackup" archive-get -B "backup_dir" --instance=instance_name --wal-file-path=%p --wal-file-name=%f --remote-host=192.168.0.3 --remote-port=2303 --remote-user=backup' @@ -2091,7 +2658,8 @@ pg_probackup restore -B backup_dir --instance , , , - and + , + , and processes can be executed on several parallel threads. This can significantly speed up pg_probackup operation given enough resources (CPU @@ -2103,7 +2671,7 @@ pg_probackup restore -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL -j 4 +pg_probackup backup -B backup_dir --instance=instance_name -b FULL -j 4 @@ -2164,14 +2732,14 @@ pg_probackup backup -B backup_dir --instance set-config command: -pg_probackup set-config -B backup_dir --instance instance_name +pg_probackup set-config -B backup_dir --instance=instance_name [--external-dirs=external_directory_path] [remote_options] [connection_options] [retention_options] [logging_options] To view the current settings, run the following command: -pg_probackup show-config -B backup_dir --instance instance_name +pg_probackup show-config -B backup_dir --instance=instance_name You can override the settings defined in pg_probackup.conf when @@ -2245,16 +2813,16 @@ pg_probackup show -B backup_dir pg_probackup displays the list of all the available backups. For example: - + BACKUP INSTANCE 'node' ====================================================================================================================================== - Instance Version ID Recovery time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status ====================================================================================================================================== - node 10 PYSUE8 2019-10-03 15:51:48+03 FULL ARCHIVE 1/0 16s 9047kB 16MB 4.31 0/12000028 0/12000160 OK - node 10 P7XDQV 2018-04-29 05:32:59+03 DELTA STREAM 1/1 11s 19MB 16MB 1.00 0/15000060 0/15000198 OK - node 10 P7XDJA 2018-04-29 05:28:36+03 PTRACK STREAM 1/1 21s 32MB 32MB 1.00 0/13000028 0/13000198 OK - node 10 P7XDHU 2018-04-29 05:27:59+03 PAGE STREAM 1/1 15s 33MB 16MB 1.00 0/11000028 0/110001D0 OK - node 10 P7XDHB 2018-04-29 05:27:15+03 FULL STREAM 1/0 11s 39MB 16MB 1.00 0/F000028 0/F000198 OK + node 16 SCUN4E 2024-05-02 11:19:37+03 FULL ARCHIVE 1/0 13s 239MB 16MB 2.31 0/4C000028 0/4D0000B8 OK + node 16 SCUN3Y 2024-05-02 11:19:16+03 DELTA STREAM 1/1 7s 92MB 208MB 2.27 0/3C0043A8 0/46159C70 OK + node 16 SCUN3M 2024-05-02 11:19:01+03 PTRACK STREAM 1/1 10s 30MB 16MB 2.23 0/32000028 0/32005ED0 OK + node 16 SCUN39 2024-05-02 11:18:50+03 PAGE STREAM 1/1 12s 46MB 32MB 1.44 0/2A000028 0/2B0000B8 OK + node 16 SCUN2V 2024-05-02 11:18:38+03 FULL STREAM 1/0 11s 154MB 16MB 2.32 0/23000028 0/23000168 OK For each backup, the following information is provided: @@ -2404,12 +2972,12 @@ BACKUP INSTANCE 'node' show command with the backup ID: -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id The sample output is as follows: - + #Configuration backup-mode = FULL stream = false @@ -2419,27 +2987,26 @@ from-replica = false #Compatibility block-size = 8192 -wal-block-size = 8192 +xlog-block-size = 8192 checksum-version = 1 -program-version = 2.1.3 -server-version = 10 +program-version = 2.5.15 +server-version = 16 #Result backup info timelineid = 1 -start-lsn = 0/04000028 -stop-lsn = 0/040000f8 -start-time = '2017-05-16 12:57:29' -end-time = '2017-05-16 12:57:31' -recovery-xid = 597 -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-05-16 12:57:31' -data-bytes = 22288792 +start-lsn = 0/4C000028 +stop-lsn = 0/4D0000B8 +start-time = '2024-05-02 11:19:26+03' +end-time = '2024-05-02 11:19:39+03' +recovery-xid = 743 +recovery-time = '2024-05-02 11:19:37+03' +data-bytes = 250827955 wal-bytes = 16777216 -uncompressed-bytes = 39961833 -pgdata-bytes = 39859393 +uncompressed-bytes = 578216425 +pgdata-bytes = 578216107 status = OK -parent-backup-id = 'PT8XFX' -primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any' +primary_conninfo = 'user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable' +content-crc = 802820606 Detailed output has additional attributes: @@ -2548,44 +3115,46 @@ primary_conninfo = 'user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmod in the JSON format: -pg_probackup show -B backup_dir --instance instance_name --format=json -i backup_id +pg_probackup show -B backup_dir --instance=instance_name --format=json -i backup_id The sample output is as follows: - + [ - { - "instance": "node", - "backups": [ - { - "id": "PT91HZ", - "parent-backup-id": "PT8XFX", - "backup-mode": "DELTA", - "wal": "ARCHIVE", - "compress-alg": "zlib", - "compress-level": 1, - "from-replica": false, - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.3", - "server-version": "10", - "current-tli": 16, - "parent-tli": 2, - "start-lsn": "0/8000028", - "stop-lsn": "0/8000160", - "start-time": "2019-06-17 18:25:11+03", - "end-time": "2019-06-17 18:25:16+03", - "recovery-xid": 0, - "recovery-time": "2019-06-17 18:25:15+03", - "data-bytes": 106733, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } + { + "instance": "node", + "backups": [ + { + "id": "SCUN4E", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 16, + "parent-tli": 2, + "start-lsn": "0/4C000028", + "stop-lsn": "0/4D0000B8", + "start-time": "2024-05-02 11:19:26+03", + "end-time": "2024-05-02 11:19:39+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:37+03", + "data-bytes": 250827955, + "wal-bytes": 16777216, + "uncompressed-bytes": 578216425, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 802820606 + } + ] + } ] @@ -2596,22 +3165,19 @@ pg_probackup show -B backup_dir --instance -pg_probackup show -B backup_dir [--instance instance_name] --archive +pg_probackup show -B backup_dir [--instance=instance_name] --archive pg_probackup displays the list of all the available WAL files grouped by timelines. For example: - + + ARCHIVE INSTANCE 'node' -=================================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=================================================================================================================================== - 5 1 0/B000000 00000005000000000000000B 00000005000000000000000C 2 685kB 48.00 0 OK - 4 3 0/18000000 000000040000000000000018 00000004000000000000001A 3 648kB 77.00 0 OK - 3 2 0/15000000 000000030000000000000015 000000030000000000000017 3 648kB 77.00 0 OK - 2 1 0/B000108 00000002000000000000000B 000000020000000000000015 5 892kB 94.00 1 DEGRADED - 1 0 0/0 000000010000000000000001 00000001000000000000000A 10 8774kB 19.00 1 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000019 00000001000000000000004D 53 848MB 1.00 5 OK For each timeline, the following information is provided: @@ -2695,219 +3261,176 @@ ARCHIVE INSTANCE 'node' format, run the command: -pg_probackup show -B backup_dir [--instance instance_name] --archive --format=json +pg_probackup show -B backup_dir [--instance=instance_name] --archive --format=json The sample output is as follows: - + [ - { - "instance": "replica", - "timelines": [ - { - "tli": 5, - "parent-tli": 1, - "switchpoint": "0/B000000", - "min-segno": "00000005000000000000000B", - "max-segno": "00000005000000000000000C", - "n-segments": 2, - "size": 685320, - "zratio": 48.00, - "closest-backup-id": "PXS92O", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 4, - "parent-tli": 3, - "switchpoint": "0/18000000", - "min-segno": "000000040000000000000018", - "max-segno": "00000004000000000000001A", - "n-segments": 3, - "size": 648625, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 3, - "parent-tli": 2, - "switchpoint": "0/15000000", - "min-segno": "000000030000000000000015", - "max-segno": "000000030000000000000017", - "n-segments": 3, - "size": 648911, - "zratio": 77.00, - "closest-backup-id": "PXS9CE", - "status": "OK", - "lost-segments": [], - "backups": [] - }, - { - "tli": 2, - "parent-tli": 1, - "switchpoint": "0/B000108", - "min-segno": "00000002000000000000000B", - "max-segno": "000000020000000000000015", - "n-segments": 5, - "size": 892173, - "zratio": 94.00, - "closest-backup-id": "PXS92O", - "status": "DEGRADED", - "lost-segments": [ - { - "begin-segno": "00000002000000000000000D", - "end-segno": "00000002000000000000000E" - }, - { - "begin-segno": "000000020000000000000010", - "end-segno": "000000020000000000000012" - } - ], - "backups": [ - { - "id": "PXS9CE", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 2, - "parent-tli": 0, - "start-lsn": "0/C000028", - "stop-lsn": "0/C000160", - "start-time": "2019-09-13 21:43:26+03", - "end-time": "2019-09-13 21:43:30+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:43:29+03", - "data-bytes": 104674852, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - }, - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "000000010000000000000001", - "max-segno": "00000001000000000000000A", - "n-segments": 10, - "size": 8774805, - "zratio": 19.00, - "closest-backup-id": "", - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92O", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "true", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/4000028", - "stop-lsn": "0/6000028", - "start-time": "2019-09-13 21:37:36+03", - "end-time": "2019-09-13 21:38:45+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 25987319, - "wal-bytes": 50331648, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - }, - { - "instance": "master", - "timelines": [ - { - "tli": 1, - "parent-tli": 0, - "switchpoint": "0/0", - "min-segno": "000000010000000000000001", - "max-segno": "00000001000000000000000B", - "n-segments": 11, - "size": 8860892, - "zratio": 20.00, - "status": "OK", - "lost-segments": [], - "backups": [ - { - "id": "PXS92H", - "parent-backup-id": "PXS92C", - "backup-mode": "PAGE", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 1, - "start-lsn": "0/4000028", - "stop-lsn": "0/50000B8", - "start-time": "2019-09-13 21:37:29+03", - "end-time": "2019-09-13 21:37:31+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:30+03", - "data-bytes": 1328461, - "wal-bytes": 33554432, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - }, - { - "id": "PXS92C", - "backup-mode": "FULL", - "wal": "ARCHIVE", - "compress-alg": "none", - "compress-level": 1, - "from-replica": "false", - "block-size": 8192, - "xlog-block-size": 8192, - "checksum-version": 1, - "program-version": "2.1.5", - "server-version": "10", - "current-tli": 1, - "parent-tli": 0, - "start-lsn": "0/2000028", - "stop-lsn": "0/2000160", - "start-time": "2019-09-13 21:37:24+03", - "end-time": "2019-09-13 21:37:29+03", - "recovery-xid": 0, - "recovery-time": "2019-09-13 21:37:28+03", - "data-bytes": 24871902, - "wal-bytes": 16777216, - "primary_conninfo": "user=backup passfile=/var/lib/pgsql/.pgpass port=5432 sslmode=disable sslcompression=1 target_session_attrs=any", - "status": "OK" - } - ] - } - ] - } + { + "instance": "node", + "timelines": [ + { + "tli": 1, + "parent-tli": 0, + "switchpoint": "0/0", + "min-segno": "000000010000000000000019", + "max-segno": "00000001000000000000004D", + "n-segments": 53, + "size": 889192448, + "zratio": 1.00, + "closest-backup-id": "", + "status": "OK", + "lost-segments": [], + "backups": [ + { + "id": "SCUN4E", + "backup-mode": "FULL", + "wal": "ARCHIVE", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/4C000028", + "stop-lsn": "0/4D0000B8", + "start-time": "2024-05-02 11:19:26+03", + "end-time": "2024-05-02 11:19:39+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:37+03", + "data-bytes": 250827955, + "wal-bytes": 16777216, + "uncompressed-bytes": 578216425, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 802820606 + }, + { + "id": "SCUN3Y", + "parent-backup-id": "SCUN3M", + "backup-mode": "DELTA", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/3C0043A8", + "stop-lsn": "0/46159C70", + "start-time": "2024-05-02 11:19:10+03", + "end-time": "2024-05-02 11:19:17+03", + "recovery-xid": 743, + "recovery-time": "2024-05-02 11:19:16+03", + "data-bytes": 96029293, + "wal-bytes": 218103808, + "uncompressed-bytes": 217639806, + "pgdata-bytes": 578216107, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3074300814 + }, + { + "id": "SCUN3M", + "parent-backup-id": "SCUN39", + "backup-mode": "PTRACK", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/32000028", + "stop-lsn": "0/32005ED0", + "start-time": "2024-05-02 11:18:58+03", + "end-time": "2024-05-02 11:19:08+03", + "recovery-xid": 742, + "recovery-time": "2024-05-02 11:19:01+03", + "data-bytes": 31205704, + "wal-bytes": 16777216, + "uncompressed-bytes": 69585790, + "pgdata-bytes": 509927595, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3446949708 + }, + { + "id": "SCUN39", + "parent-backup-id": "SCUN2V", + "backup-mode": "PAGE", + "wal": "STREAM", + "compress-alg": "pglz", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 1, + "parent-tli": 1, + "start-lsn": "0/2A000028", + "stop-lsn": "0/2B0000B8", + "start-time": "2024-05-02 11:18:45+03", + "end-time": "2024-05-02 11:18:57+03", + "recovery-xid": 741, + "recovery-time": "2024-05-02 11:18:50+03", + "data-bytes": 48381612, + "wal-bytes": 33554432, + "uncompressed-bytes": 69569406, + "pgdata-bytes": 441639083, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 3492989773 + }, + { + "id": "SCUN2V", + "backup-mode": "FULL", + "wal": "STREAM", + "compress-alg": "zlib", + "compress-level": 1, + "from-replica": "false", + "block-size": 8192, + "xlog-block-size": 8192, + "checksum-version": 1, + "program-version": "2.5.15", + "server-version": "16", + "current-tli": 1, + "parent-tli": 0, + "start-lsn": "0/23000028", + "stop-lsn": "0/23000168", + "start-time": "2024-05-02 11:18:31+03", + "end-time": "2024-05-02 11:18:42+03", + "recovery-xid": 740, + "recovery-time": "2024-05-02 11:18:38+03", + "data-bytes": 161084290, + "wal-bytes": 16777216, + "uncompressed-bytes": 373359081, + "pgdata-bytes": 373358763, + "primary_conninfo": "user=backup channel_binding=prefer host=localhost port=5432 sslmode=prefer sslcompression=0 sslcertmode=allow sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=prefer krbsrvname=postgres gssdelegation=0 target_session_attrs=any load_balance_hosts=disable", + "status": "OK", + "content-crc": 1621343133 + } + ] + } + ] + } ] @@ -2997,7 +3520,7 @@ pg_probackup show -B backup_dir [--instance -pg_probackup set-config -B backup_dir --instance instance_name --retention-redundancy=2 --retention-window=7 +pg_probackup set-config -B backup_dir --instance=instance_name --retention-redundancy=2 --retention-window=7 @@ -3015,7 +3538,7 @@ pg_probackup set-config -B backup_dir --instance --delete-expired flag: -pg_probackup delete -B backup_dir --instance instance_name --delete-expired +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired If you would like to also remove the WAL files that are no @@ -3023,7 +3546,7 @@ pg_probackup delete -B backup_dir --instance --delete-wal flag: -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --delete-wal +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --delete-wal @@ -3034,7 +3557,7 @@ pg_probackup delete -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --retention-window=7 --retention-redundancy=2 +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --retention-window=7 --retention-redundancy=2 Since incremental backups require that their parent full @@ -3054,48 +3577,48 @@ pg_probackup delete -B backup_dir --instance backup_dir directory, with the option set to 7, and you have the following backups - available on April 10, 2019: + available on May 02, 2024: - + BACKUP INSTANCE 'node' -=================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -=================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 DELTA STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK - -------------------------------------------------------retention window-------------------------------------------------------- - node 10 P7XDHU 2019-04-02 05:27:59+03 PAGE STREAM 1/0 31s 33MB 16MB 1.0 0/11000028 0/110001D0 OK - node 10 P7XDHB 2019-04-01 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/F000028 0/F000198 OK - node 10 P7XDFT 2019-03-29 05:26:25+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/D000028 0/D000198 OK +===================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +===================================================================================================================================== + node 16 SCUN6L 2024-05-02 11:20:48+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/46000028 0/470000B8 OK + node 16 SCQXUI 2024-04-30 11:20:45+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/44000028 0/450000F0 OK + node 16 SCFTUG 2024-04-24 11:20:43+03 DELTA ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/42000028 0/430000B8 OK +----------------------------------------------------------retention window----------------------------------------------------------- + node 16 SCDZ6D 2024-04-23 11:20:40+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/40000028 0/410000B8 OK + node 16 SCC4HX 2024-04-22 11:20:24+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3E000028 0/3F0000F0 OK + node 16 SC8F5G 2024-04-20 11:20:07+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/3C0000D8 0/3D00BB58 OK - Even though P7XDHB and P7XDHU backups are outside the + Even though SCC4HX and SCDZ6D backups are outside the retention window, they cannot be removed as it invalidates the - succeeding incremental backups P7XDJA and P7XDQV that are + succeeding incremental backups SCFTUG and SCQXUI that are still required, so, if you run the command with the - flag, only the P7XDFT full + flag, only the SC8F5G full backup will be removed. - With the option, the P7XDJA - backup is merged with the underlying P7XDHU and P7XDHB backups + With the option, the SCFTUG + backup is merged with the underlying SCDZ6D and SCC4HX backups and becomes a full one, so there is no need to keep these expired backups anymore: -pg_probackup delete -B backup_dir --instance node --delete-expired --merge-expired +pg_probackup delete -B backup_dir --instance=node --delete-expired --merge-expired pg_probackup show -B backup_dir - + BACKUP INSTANCE 'node' -================================================================================================================================== - Instance Version ID Recovery time Mode WAL TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 10 P7XDHR 2019-04-10 05:27:15+03 FULL STREAM 1/0 11s 200MB 16MB 1.0 0/18000059 0/18000197 OK - node 10 P7XDQV 2019-04-08 05:32:59+03 PAGE STREAM 1/0 11s 19MB 16MB 1.0 0/15000060 0/15000198 OK - node 10 P7XDJA 2019-04-03 05:28:36+03 FULL STREAM 1/0 21s 32MB 16MB 1.0 0/13000028 0/13000198 OK +===================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +===================================================================================================================================== + node 16 SCUN6L 2024-05-02 11:20:48+03 FULL ARCHIVE 1/0 5s 296MB 16MB 2.30 0/46000028 0/470000B8 OK + node 16 SCQXUI 2024-04-30 11:20:45+03 PAGE ARCHIVE 1/1 5s 6280kB 16MB 1.00 0/44000028 0/450000F0 OK + node 16 SCFTUG 2024-04-24 11:20:43+03 FULL ARCHIVE 1/1 5s 296MB 16MB 1.00 0/42000028 0/430000B8 OK The Time field for the merged backup displays the time @@ -3111,7 +3634,7 @@ BACKUP INSTANCE 'node' for arbitrary time. For example: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=30d +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --ttl=30d This command sets the expiration time of the @@ -3123,7 +3646,7 @@ pg_probackup set-backup -B backup_dir --instance --expire-time option. For example: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --expire-time='2020-01-01 00:00:00+03' +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --expire-time="2027-05-02 11:21:00+00" Alternatively, you can use the and @@ -3132,23 +3655,23 @@ pg_probackup set-backup -B backup_dir --instance -pg_probackup backup -B backup_dir --instance instance_name -b FULL --ttl=30d -pg_probackup backup -B backup_dir --instance instance_name -b FULL --expire-time='2020-01-01 00:00:00+03' +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --ttl=30d +pg_probackup backup -B backup_dir --instance=instance_name -b FULL --expire-time="2027-05-02 11:21:00+00" To check if the backup is pinned, run the command: -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id If the backup is pinned, it has the expire-time attribute that displays its expiration time: - + ... -recovery-time = '2017-05-16 12:57:31' -expire-time = '2020-01-01 00:00:00+03' +recovery-time = '2024-05-02 11:21:00+00' +expire-time = '2027-05-02 11:21:00+00' data-bytes = 22288792 ... @@ -3157,7 +3680,7 @@ data-bytes = 22288792 You can unpin the backup by setting the option to zero: -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id --ttl=0 +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id --ttl=0 @@ -3221,19 +3744,18 @@ pg_probackup set-backup -B backup_dir --instance : -pg_probackup show -B backup_dir --instance node +pg_probackup show -B backup_dir --instance=node - -BACKUP INSTANCE 'node' -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ9442 2019-10-12 10:43:21+03 DELTA STREAM 1/0 10s 121kB 16MB 1.00 0/46000028 0/46000160 OK - node 11 PZ943L 2019-10-12 10:43:04+03 FULL STREAM 1/0 10s 180MB 32MB 1.00 0/44000028 0/44000160 OK - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - node 11 PZ7YFO 2019-10-11 19:43:04+03 FULL STREAM 1/0 10s 30MB 16MB 1.00 0/2000028 0/200ADD8 OK + +====================================================================================================================================== + Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status +====================================================================================================================================== + node 16 SCUN92 2024-05-02 11:22:16+03 DELTA STREAM 1/1 9s 1162kB 32MB 1.08 0/7C000028 0/7C000168 OK + node 16 SCUN8N 2024-05-02 11:22:09+03 FULL STREAM 1/0 12s 296MB 16MB 2.30 0/7A000028 0/7A009A08 OK + node 16 SCUN8I 2024-05-02 11:21:55+03 DELTA STREAM 1/1 5s 1148kB 32MB 1.01 0/78000028 0/78000168 OK + node 16 SCUN86 2024-05-02 11:21:47+03 DELTA STREAM 1/1 11s 120MB 16MB 2.27 0/76000028 0/760001A0 OK + node 16 SCUN7I 2024-05-02 11:21:29+03 FULL STREAM 1/0 22s 296MB 288MB 2.30 0/63012FE8 0/74E7ADA0 OK + node 16 SCUN71 2024-05-02 11:21:12+03 FULL STREAM 1/0 13s 296MB 272MB 2.30 0/49000028 0/573683B8 OK You can check the state of the WAL archive by running the @@ -3241,28 +3763,30 @@ BACKUP INSTANCE 'node' flag: -pg_probackup show -B backup_dir --instance node --archive +pg_probackup show -B backup_dir --instance=node --archive - + + ARCHIVE INSTANCE 'node' -=============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================================== - 1 0 0/0 000000010000000000000001 000000010000000000000047 71 36MB 31.00 6 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000048 00000001000000000000007C 53 848MB 1.00 6 OK WAL purge without cannot achieve much, only one segment is removed: -pg_probackup delete -B backup_dir --instance node --delete-wal +pg_probackup delete -B backup_dir --instance=node --delete-wal - + + ARCHIVE INSTANCE 'node' -=============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -=============================================================================================================================== - 1 0 0/0 000000010000000000000002 000000010000000000000047 70 34MB 32.00 6 OK +================================================================================================================================ + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +================================================================================================================================ + 1 0 0/0 000000010000000000000049 00000001000000000000007C 52 832MB 1.00 6 OK If you would like, for example, to keep only those WAL @@ -3270,28 +3794,30 @@ ARCHIVE INSTANCE 'node' option to 1: -pg_probackup delete -B backup_dir --instance node --delete-wal --wal-depth=1 +pg_probackup delete -B backup_dir --instance=node --delete-wal --wal-depth=1 - + + ARCHIVE INSTANCE 'node' -================================================================================================================================ - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status -================================================================================================================================ - 1 0 0/0 000000010000000000000046 000000010000000000000047 2 143kB 228.00 6 OK +=============================================================================================================================== + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status +=============================================================================================================================== + 1 0 0/0 00000001000000000000007C 00000001000000000000007C 1 16MB 1.00 6 OK Alternatively, you can use the option with the command: -pg_probackup backup -B backup_dir --instance node -b DELTA --wal-depth=1 --delete-wal +pg_probackup backup -B backup_dir --instance=node -b DELTA --wal-depth=1 --delete-wal - + + ARCHIVE INSTANCE 'node' =============================================================================================================================== - TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status + TLI Parent TLI Switchpoint Min Segno Max Segno N segments Size Zratio N backups Status =============================================================================================================================== - 1 0 0/0 000000010000000000000048 000000010000000000000049 1 72kB 228.00 7 OK + 1 0 0/0 00000001000000000000007E 00000001000000000000007E 1 16MB 1.00 7 OK @@ -3305,7 +3831,7 @@ ARCHIVE INSTANCE 'node' recent incremental backup you would like to merge: -pg_probackup merge -B backup_dir --instance instance_name -i backup_id +pg_probackup merge -B backup_dir --instance=instance_name -i backup_id This command merges backups that belong to a common incremental backup @@ -3315,7 +3841,7 @@ pg_probackup merge -B backup_dir --instance pg_probackup in the remote mode. @@ -3327,7 +3853,7 @@ pg_probackup merge -B backup_dir --instance -pg_probackup show -B backup_dir --instance instance_name -i backup_id +pg_probackup show -B backup_dir --instance=instance_name -i backup_id If the merge is still in progress, the backup status is @@ -3345,69 +3871,243 @@ pg_probackup show -B backup_dir --instance -pg_probackup delete -B backup_dir --instance instance_name -i backup_id +pg_probackup delete -B backup_dir --instance=instance_name -i backup_id + + + This command will delete the backup with the specified + backup_id, together with all the + incremental backups that descend from + backup_id, if any. This way you can delete + some recent incremental backups, retaining the underlying full + backup and some of the incremental backups that follow it. + + + To delete obsolete WAL files that are not necessary to restore + any of the remaining backups, use the + flag: + + +pg_probackup delete -B backup_dir --instance=instance_name --delete-wal + + + To delete backups that are expired according to the current + retention policy, use the + flag: + + +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired + + + Expired backups cannot be removed while at least one + incremental backup that satisfies the retention policy is based + on them. If you would like to minimize the number of backups + still required to keep incremental backups valid, specify the + flag when running this + command: + + +pg_probackup delete -B backup_dir --instance=instance_name --delete-expired --merge-expired + + + In this case, pg_probackup searches for the oldest incremental + backup that satisfies the retention policy and merges this + backup with the underlying full and incremental backups that + have already expired, thus making it a full backup. Once the + merge is complete, the remaining expired backups are deleted. + + + Before merging or deleting backups, you can run the + delete command with the + flag, which displays the status of + all the available backups according to the current retention + policy, without performing any irreversible actions. + + + To delete all backups with specific status, use the : + + +pg_probackup delete -B backup_dir --instance=instance_name --status=ERROR + + + + Deleting backups by status ignores established retention policies. + + + + + + Cloning and Synchronizing <productname>PostgreSQL</productname> Instance + + pg_probackup can create a copy of a PostgreSQL + instance directly, without using the backup catalog. To do this, you can run the command. + It can be useful in the following cases: + + + To add a new standby server. + Usually, pg_basebackup + is used to create a copy of a PostgreSQL instance. If the data directory of the destination instance + is empty, the catchup command works similarly, but it can be faster if run in parallel mode. + + + To have a fallen-behind standby server catch up with master. + Under write-intensive load, replicas may fail to replay WAL fast enough to keep up with master and hence may lag behind. + A usual solution to create a new replica and switch to it requires a lot of extra space and data transfer. The catchup + command allows you to update an existing replica much faster by bringing differences from master. + + + + + + catchup is different from other pg_probackup + operations: + + + + The backup catalog is not required. + + + + + STREAM WAL delivery mode is only supported. + + + + + Copying external directories + is not supported. + + + + + DDL commands + CREATE TABLESPACE/DROP TABLESPACE + cannot be run simultaneously with catchup. + + + + + catchup takes configuration files, such as + postgresql.conf, postgresql.auto.conf, + or pg_hba.conf, from the source server and overwrites them + on the target server. The option allows you to keep + the configuration files intact. + + + + + + + To prepare for cloning/synchronizing a PostgreSQL instance, + set up the source server as follows: + + + + Configure + the database cluster for the instance to copy. + + + + + To copy from a remote server, configure the remote mode. + + + + + To use the PTRACK catchup mode, set up PTRACK backups. + + + + + + + Before cloning/synchronizing a PostgreSQL instance, ensure that the source + server is running and accepting connections. To clone/sync a PostgreSQL instance, + on the server with the destination instance, you can run + the command as follows: + + +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream [connection_options] [remote_options] - This command will delete the backup with the specified - backup_id, together with all the - incremental backups that descend from - backup_id, if any. This way you can delete - some recent incremental backups, retaining the underlying full - backup and some of the incremental backups that follow it. + Where catchup_mode can take one of the + following values: + + + + FULL — creates a full copy of the PostgreSQL instance. + The data directory of the destination instance must be empty for this mode. + + + + + DELTA — reads all data files in the data directory and + creates an incremental copy for pages that have changed + since the destination instance was shut down. + + + + + PTRACK — tracking page changes on the fly, + only reads and copies pages that have changed since the point of divergence + of the source and destination instances. + + + PTRACK catchup mode requires PTRACK + not earlier than 2.0 and hence, PostgreSQL not earlier than 11. + + + + + - To delete obsolete WAL files that are not necessary to restore - any of the remaining backups, use the - flag: + By specifying the option, you can set + STREAM WAL delivery mode + of copying, which will include all the necessary WAL files by streaming them from + the server via replication protocol. - -pg_probackup delete -B backup_dir --instance instance_name --delete-wal - - To delete backups that are expired according to the current - retention policy, use the - flag: + You can use connection_options to specify + the connection to the source database cluster. If it is located on a different server, + also specify remote_options. - -pg_probackup delete -B backup_dir --instance instance_name --delete-expired - - Expired backups cannot be removed while at least one - incremental backup that satisfies the retention policy is based - on them. If you would like to minimize the number of backups - still required to keep incremental backups valid, specify the - flag when running this - command: - - -pg_probackup delete -B backup_dir --instance instance_name --delete-expired --merge-expired + If the source database cluster contains tablespaces that must be located in + a different directory, additionally specify the + option: + +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --tablespace-mapping=OLDDIR=NEWDIR + + To run the catchup command on parallel threads, specify the number + of threads with the option: + +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --threads=num_threads - - In this case, pg_probackup searches for the oldest incremental - backup that satisfies the retention policy and merges this - backup with the underlying full and incremental backups that - have already expired, thus making it a full backup. Once the - merge is complete, the remaining expired backups are deleted. - Before merging or deleting backups, you can run the - delete command with the - flag, which displays the status of - all the available backups according to the current retention - policy, without performing any irreversible actions. + Before cloning/synchronising a PostgreSQL instance, you can run the + catchup command with the flag + to estimate the size of data files to be transferred, but make no changes on disk: + +pg_probackup catchup -b catchup_mode --source-pgdata=path_to_pgdata_on_remote_server --destination-pgdata=path_to_local_dir --stream --dry-run + - To delete all backups with specific status, use the : + For example, assume that a remote standby server with the PostgreSQL instance having /replica-pgdata data directory has fallen behind. To sync this instance with the one in /master-pgdata data directory, you can run + the catchup command in the PTRACK mode on four parallel threads as follows: + +pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=PTRACK --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 --exclude-path=postgresql.conf --exclude-path=postgresql.auto.conf --exclude-path=pg_hba.conf --exclude-path=pg_ident.conf + + Note that in this example, the configuration files will not be overwritten during synchronization. - -pg_probackup delete -B backup_dir --instance instance_name --status=ERROR - - - Deleting backups by status ignores established retention policies. + Another example shows how you can add a new remote standby server with the PostgreSQL data directory /replica-pgdata by running the catchup command in the FULL mode + on four parallel threads: + +pg_probackup catchup --source-pgdata=/master-pgdata --destination-pgdata=/replica-pgdata -p 5432 -d postgres -U remote-postgres-user --stream --backup-mode=FULL --remote-host=remote-hostname --remote-user=remote-unix-username -j 4 + - - + @@ -3461,7 +4161,7 @@ pg_probackup init -B backup_dir [--help] add-instance -pg_probackup add-instance -B backup_dir -D data_dir --instance instance_name [--help] +pg_probackup add-instance -B backup_dir -D data_dir --instance=instance_name [--help] Initializes a new backup instance inside the backup catalog @@ -3479,7 +4179,7 @@ pg_probackup add-instance -B backup_dir -D del-instance -pg_probackup del-instance -B backup_dir --instance instance_name [--help] +pg_probackup del-instance -B backup_dir --instance=instance_name [--help] Deletes all backups and WAL files associated with the @@ -3489,7 +4189,7 @@ pg_probackup del-instance -B backup_dir --instance set-config -pg_probackup set-config -B backup_dir --instance instance_name +pg_probackup set-config -B backup_dir --instance=instance_name [--help] [--pgdata=pgdata-path] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--compress-algorithm=compression_algorithm] [--compress-level=compression_level] @@ -3515,7 +4215,7 @@ pg_probackup set-config -B backup_dir --instance set-backup -pg_probackup set-backup -B backup_dir --instance instance_name -i backup_id +pg_probackup set-backup -B backup_dir --instance=instance_name -i backup_id {--ttl=ttl | --expire-time=time} [--note=backup_note] [--help] @@ -3546,7 +4246,8 @@ pg_probackup set-backup -B backup_dir --instance show-config -pg_probackup show-config -B backup_dir --instance instance_name [--format=plain|json] +pg_probackup show-config -B backup_dir --instance=instance_name [--format=plain|json] +[--no-scale-units] [logging_options] Displays the contents of the pg_probackup.conf configuration @@ -3557,6 +4258,18 @@ pg_probackup show-config -B backup_dir --instance JSON format. By default, configuration settings are shown as plain text. + + You can also specify the + option to display time and memory configuration settings in their base (unscaled) units. + Otherwise, the values are scaled to larger units for optimal display. + For example, if archive-timeout is 300, then + 5min is displayed, but if archive-timeout + is 301, then 301s is displayed. + Also, if the option is specified, configuration + settings are displayed without units and for the JSON format, + numeric and boolean values are not enclosed in quotes. This facilitates parsing + the output. + To edit pg_probackup.conf, use the command. @@ -3566,7 +4279,7 @@ pg_probackup show-config -B backup_dir --instance show pg_probackup show -B backup_dir -[--help] [--instance instance_name [-i backup_id | --archive]] [--format=plain|json] +[--help] [--instance=instance_name [-i backup_id | --archive]] [--format=plain|json] [--no-color] Shows the contents of the backup catalog. If @@ -3581,6 +4294,8 @@ pg_probackup show -B backup_dir plain text. You can specify the --format=json option to get the result in the JSON format. + If --no-color flag is used, + then the output is not colored. For details on usage, see the sections @@ -3593,7 +4308,7 @@ pg_probackup show -B backup_dir backup -pg_probackup backup -B backup_dir -b backup_mode --instance instance_name +pg_probackup backup -B backup_dir -b backup_mode --instance=instance_name [--help] [-j num_threads] [--progress] [-C] [--stream [-S slot_name] [--temp-slot]] [--backup-pg-log] [--no-validate] [--skip-block-validation] @@ -3613,35 +4328,10 @@ pg_probackup backup -B backup_dir -b bac Specifies the backup mode to use. Possible values are: - - - - - FULL — creates a full backup that contains all the data - files of the cluster to be restored. - - - - - DELTA — reads all data files in the data directory and - creates an incremental backup for pages that have changed - since the previous backup. - - - - - PAGE — creates an incremental PAGE backup based on the WAL - files that have changed since the previous full or - incremental backup was taken. - - - - - PTRACK — creates an incremental PTRACK backup tracking - page changes on the fly. - - - + FULL, + DELTA, + PAGE, and + PTRACK. @@ -3801,7 +4491,7 @@ pg_probackup backup -B backup_dir -b bac restore -pg_probackup restore -B backup_dir --instance instance_name +pg_probackup restore -B backup_dir --instance=instance_name [--help] [-D data_dir] [-i backup_id] [-j num_threads] [--progress] [-T OLDDIR=NEWDIR] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] @@ -3810,6 +4500,7 @@ pg_probackup restore -B backup_dir --instance cmdline] [--primary-conninfo=primary_conninfo] [-S | --primary-slot-name=slot_name] +[-X wal_dir | --waldir=wal_dir] [recovery_target_options] [logging_options] [remote_options] [partial_restore_options] [remote_wal_archive_options] @@ -3857,7 +4548,7 @@ pg_probackup restore -B backup_dir --instance -R flag is specified. - Example: --primary-conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' + Example: --primary-conninfo="host=192.168.1.50 port=5432 user=foo password=foopass" @@ -3956,6 +4647,11 @@ pg_probackup restore -B backup_dir --instance PostgreSQL cluster from a corrupt or an invalid backup. Use with caution. + If PGDATA contains a non-empty directory with system ID different from that + of the backup being restored, incremental restore + with this flag overwrites the directory contents (while an error occurs without the flag). If tablespaces + are remapped through the --tablespace-mapping option into non-empty directories, + the contents of such directories will be deleted. @@ -3972,6 +4668,17 @@ pg_probackup restore -B backup_dir --instance + + + + + + + Specifies the directory where WAL should be stored. + + + + @@ -3995,9 +4702,9 @@ pg_probackup restore -B backup_dir --instance checkdb pg_probackup checkdb -[-B backup_dir] [--instance instance_name] [-D data_dir] +[-B backup_dir] [--instance=instance_name] [-D data_dir] [--help] [-j num_threads] [--progress] -[--skip-block-validation] [--amcheck] [--heapallindexed] +[--amcheck [--skip-block-validation] [--checkunique] [--heapallindexed]] [connection_options] [logging_options] @@ -4012,21 +4719,30 @@ pg_probackup checkdb Performs logical verification of indexes for the specified PostgreSQL instance if no corruption was found while checking - data files. You must have the amcheck + data files. You must have the amcheck extension or the amcheck_next extension installed in the database to check its indexes. For databases without amcheck, index verification will be skipped. + Additional options and + are effective depending on the version of amcheck installed. - + - Skip validation of data files. You can use this flag only - together with the flag, so that only logical - verification of indexes is performed. + Verifies unique constraints during logical verification of indexes. + You can use this flag only together with the flag when + the amcheck extension is + installed in the database. + + + The verification of unique constraints is only possible if in the version of the + amcheck extension you are using, the + bt_index_check function takes the + checkunique parameter. @@ -4040,12 +4756,25 @@ pg_probackup checkdb flag. - This check is only possible if you are using the - amcheck extension of version 2.0 or higher, or - the amcheck_next extension of any version. + This check is only possible if in the version of the + amcheck/amcheck_next extension + you are using, the bt_index_check + function takes the heapallindexed parameter. + + + + + + + + + Skip validation of data files. You can use this flag only + together with the flag, so that only logical + verification of indexes is performed. + @@ -4063,7 +4792,7 @@ pg_probackup checkdb validate pg_probackup validate -B backup_dir -[--help] [--instance instance_name] [-i backup_id] +[--help] [--instance=instance_name] [-i backup_id] [-j num_threads] [--progress] [--skip-block-validation] [recovery_target_options] [logging_options] @@ -4091,8 +4820,8 @@ pg_probackup validate -B backup_dir merge -pg_probackup merge -B backup_dir --instance instance_name -i backup_id -[--help] [-j num_threads] [--progress] +pg_probackup merge -B backup_dir --instance=instance_name -i backup_id +[--help] [-j num_threads] [--progress] [--no-validate] [--no-sync] [logging_options] @@ -4103,6 +4832,30 @@ pg_probackup merge -B backup_dir --instance + + + + + + + + Skips automatic validation before and after merge. + + + + + + + + Do not sync merged files to disk. You can use this flag to speed + up the merge process. Using this flag can result in data + corruption in case of operating system or hardware crash. + + + + + + For details, see the section Merging Backups. @@ -4111,21 +4864,46 @@ pg_probackup merge -B backup_dir --instance delete -pg_probackup delete -B backup_dir --instance instance_name +pg_probackup delete -B backup_dir --instance=instance_name [--help] [-j num_threads] [--progress] [--retention-redundancy=redundancy][--retention-window=window][--wal-depth=wal_depth] [--delete-wal] {-i backup_id | --delete-expired [--merge-expired] | --merge-expired | --status=backup_status} -[--dry-run] [logging_options] +[--dry-run] [--no-validate] [--no-sync] [logging_options] Deletes backup with specified backup_id or launches the retention purge of backups and archived WAL that do not satisfy the current retention policies. + + + + + + + + Skips automatic validation before and after retention merge. + + + + + + + + + Do not sync merged files to disk. You can use this flag to speed + up the retention merge process. Using this flag can result in data + corruption in case of operating system or hardware crash. + + + + + + For details, see the sections Deleting Backups, - Retention Options and + Retention Options, and Configuring Retention Policy. @@ -4133,7 +4911,7 @@ pg_probackup delete -B backup_dir --instance archive-push -pg_probackup archive-push -B backup_dir --instance instance_name +pg_probackup archive-push -B backup_dir --instance=instance_name --wal-file-name=wal_file_name [--wal-file-path=wal_file_path] [--help] [--no-sync] [--compress] [--no-ready-rename] [--overwrite] [-j num_threads] [--batch-size=batch_size] @@ -4199,29 +4977,203 @@ pg_probackup archive-push -B backup_dir --instance archive-get -pg_probackup archive-get -B backup_dir --instance instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name +pg_probackup archive-get -B backup_dir --instance=instance_name --wal-file-path=wal_file_path --wal-file-name=wal_file_name [-j num_threads] [--batch-size=batch_size] [--prefetch-dir=prefetch_dir_path] [--no-validate-wal] [--help] [remote_options] [logging_options] - Copies WAL files from the corresponding subdirectory of the - backup catalog to the cluster's write-ahead log location. This - command is automatically set by pg_probackup as part of the - restore_command when - restoring backups using a WAL archive. You do not need to set - it manually. + Copies WAL files from the corresponding subdirectory of the + backup catalog to the cluster's write-ahead log location. This + command is automatically set by pg_probackup as part of the + restore_command when + restoring backups using a WAL archive. You do not need to set + it manually. + + + + To speed up recovery, you can specify the option + to copy WAL segments in batches of the specified size. + If option is used, then you can also specify + the option to copy the batch of WAL segments on multiple threads. + + + + For details, see section Archiving Options. + + + + + catchup + +pg_probackup catchup -b catchup_mode +--source-pgdata=path_to_pgdata_on_remote_server +--destination-pgdata=path_to_local_dir +[--help] [-j | --threads=num_threads] [--stream] [--dry-run] +[--temp-slot] [-P | --perm-slot] [-S | --slot=slot_name] +[--exclude-path=PATHNAME] +[-T OLDDIR=NEWDIR] +[connection_options] [remote_options] + + + Creates a copy of a PostgreSQL + instance without using the backup catalog. + + + + + + + + Specifies the catchup mode to use. Possible values are: + FULL, + DELTA, and + PTRACK. + + + + + + + + + Specifies the path to the data directory of the instance to be copied. The path can be local or remote. + + + + + + + + + Specifies the path to the local data directory to copy to. + + + + + + + + + + Sets the number of parallel threads for + catchup process. + + + + + + + + + Copies the instance in STREAM WAL delivery mode, + including all the necessary WAL files by streaming them from + the server via replication protocol. + + + + + + + + + Displays the total size of the files to be transferred by catchup. + This flag initiates a trial run of catchup, which does + not actually create, delete or move files on disk. WAL streaming is skipped with . + This flag also allows you to check that + all the options are correct and cloning/synchronising is ready to run. + + + + + +=path_prefix +=path_prefix + + + Specifies a prefix for files to exclude from the synchronization of PostgreSQL + instances during copying. The prefix must contain a path relative to the data directory of an instance. + If the prefix specifies a directory, + all files in this directory will not be synchronized. + + + This option is dangerous since excluding files from synchronization can result in + incomplete synchronization; use with care. + + + + + + + + + + + Creates a temporary physical replication slot for streaming + WAL from the PostgreSQL instance being copied. It ensures that + all the required WAL segments remain available if WAL is + rotated while the backup is in progress. This flag can only be + used together with the flag and + cannot be used together with the flag. + The default slot name is pg_probackup_slot, + which can be changed using the / option. + + + + + + + + + + Creates a permanent physical replication slot for streaming + WAL from the PostgreSQL instance being copied. This flag can only be + used together with the flag and + cannot be used together with the flag. + The default slot name is pg_probackup_perm_slot, + which can be changed using the / option. + + + + + + + + + + Specifies the replication slot for WAL streaming. This option + can only be used together with the + flag. + + + + + + + + + + Relocates the tablespace from the OLDDIR to the NEWDIR + directory at the time of recovery. Both OLDDIR and NEWDIR must + be absolute paths. If the path contains the equals sign (=), + escape it with a backslash. This option can be specified + multiple times for multiple tablespaces. + + - - To speed up recovery, you can specify the option - to copy WAL segments in batches of the specified size. - If option is used, then you can also specify - the option to copy the batch of WAL segments on multiple threads. + - For details, see section Archiving Options. + Additionally, connection + options, remote + mode options can be used. + + + For details on usage, see the section + Cloning and Synchronizing PostgreSQL Instance. @@ -4381,8 +5333,7 @@ pg_probackup archive-get -B backup_dir --instance Specifies the LSN of the write-ahead log location up to which - recovery will proceed. Can be used only when restoring - a database cluster of major version 10 or higher. + recovery will proceed. @@ -4401,6 +5352,10 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which recovery will proceed. + If the time zone offset is not specified, the local time zone is used. + + + Example: --recovery-target-time="2027-05-02 11:21:00+00" @@ -4587,9 +5542,10 @@ pg_probackup archive-get -B backup_dir --instance Specifies the timestamp up to which the backup will stay pinned. Must be an ISO-8601 complaint timestamp. + If the time zone offset is not specified, the local time zone is used. - Example: --expire-time='2020-01-01 00:00:00+03' + Example: --expire-time="2027-05-02 11:21:00+00" @@ -4603,6 +5559,16 @@ pg_probackup archive-get -B backup_dir --instance + + + + + + Disable coloring for console log messages of warning and error levels. + + + + @@ -4712,6 +5678,60 @@ pg_probackup archive-get -B backup_dir --instance + + + + Defines the format of the console log. Only set from the command line. Note that you cannot + specify this option in the pg_probackup.conf configuration file through + the command and that the + command also treats this option specified in the configuration file as an error. + Possible values are: + + + + + plain — sets the plain-text format of the console log. + + + + + json — sets the JSON format of the console log. + + + + + + Default: plain + + + + + + + + + Defines the format of log files used. Possible values are: + + + + + plain — sets the plain-text format of log files. + + + + + json — sets the JSON format of log files. + + + + + + Default: plain + + + + + @@ -4751,7 +5771,8 @@ pg_probackup archive-get -B backup_dir --instance Connection Options You can use these options together with - and + , + , and commands. @@ -4927,7 +5948,9 @@ pg_probackup archive-get -B backup_dir --instance archive_command and restore_command. Use the %p - variable as the value for this option for correct processing. + variable as the value for this option or explicitly specify the path to a file + outside of the data directory. If you skip this option, the path + specified in pg_probackup.conf will be used. @@ -4940,6 +5963,8 @@ pg_probackup archive-get -B backup_dir --instance archive_command and restore_command. Use the %f variable as the value for this option for correct processing. + If the value of is a path + outside of the data directory, explicitly specify the filename. @@ -5042,6 +6067,7 @@ pg_probackup archive-get -B backup_dir --instance , , , + , , , and commands. @@ -5132,7 +6158,7 @@ pg_probackup archive-get -B backup_dir --instance keep-alive for SSH connections opened by pg_probackup: - --ssh-options='-o ServerAliveCountMax=5 -o ServerAliveInterval=60'. + --ssh-options="-o ServerAliveCountMax=5 -o ServerAliveInterval=60". For the full list of possible options, see ssh_config manual page. @@ -5269,349 +6295,9 @@ pg_probackup archive-get -B backup_dir --instance - - Replica Options - - This section describes the options related to taking a backup - from standby. - - - - Starting from pg_probackup 2.0.24, backups can be - taken from standby without connecting to the master server, - so these options are no longer required. In lower versions, - pg_probackup had to connect to the master to determine - recovery time — the earliest moment for which you can - restore a consistent state of the database cluster. - - - - - - - - - Deprecated. Specifies the name of the database on the master - server to connect to. The connection is used only for managing - the backup process, so you can connect to any existing - database. Can be set in the pg_probackup.conf using the - command. - - - Default: postgres, the default PostgreSQL database - - - - - - - - - Deprecated. Specifies the host name of the system on which the - master server is running. - - - - - - - - - Deprecated. Specifies the TCP port or the local Unix domain - socket file extension on which the master server is listening - for connections. - - - Default: 5432, the PostgreSQL default port - - - - - - - - - Deprecated. User name to connect as. - - - Default: postgres, - the PostgreSQL default user name - - - - - - - - - - Deprecated. Wait time for WAL segment streaming via - replication, in seconds. By default, pg_probackup waits 300 - seconds. You can also define this parameter in the - pg_probackup.conf configuration file using the - command. - - - Default: 300 sec - - - - - - - - How-To - - All examples below assume the remote mode of operations via - SSH. If you are planning to run backup and - restore operation locally, skip the - Setup passwordless SSH connection step - and omit all options. - - - Examples are based on Ubuntu 18.04, - PostgreSQL 11, and pg_probackup - 2.2.0. - - - - - backup — PostgreSQL - role used for connection to PostgreSQL - cluster. - - - - - backupdb — database used for connection - to PostgreSQL cluster. - - - - - backup_host — host with backup catalog. - - - - - backupman — user on - backup_host running all pg_probackup - operations. - - - - - /mnt/backups — directory on - backup_host where backup catalog is stored. - - - - - postgres_host — host with PostgreSQL - cluster. - - - - - postgres — user on - postgres_host that has started the PostgreSQL cluster. - - - - - /var/lib/postgresql/11/main — PostgreSQL - data directory on postgres_host. - - - - - Minimal Setup - - This scenario illustrates setting up standalone FULL and DELTA backups. - - - - Set up passwordless SSH connection from - <literal>backup_host</literal> to - <literal>postgres_host</literal>: - -[backupman@backup_host] ssh-copy-id postgres@postgres_host - - - - Configure your <productname>PostgreSQL</productname> cluster. - - For security purposes, it is recommended to use a separate - database for backup operations. - - -postgres=# -CREATE DATABASE backupdb; - - - Connect to the backupdb database, create the - probackup role, and grant the following - permissions to this role: - - -backupdb=# -BEGIN; -CREATE ROLE backup WITH LOGIN REPLICATION; -GRANT USAGE ON SCHEMA pg_catalog TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; -GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO backup; -COMMIT; - - - - Initialize the backup catalog: - -[backupman@backup_host]$ pg_probackup-11 init -B /mnt/backups -INFO: Backup catalog '/mnt/backups' successfully inited - - - - Add instance <literal>pg-11</literal> to the backup catalog: - -[backupman@backup_host]$ pg_probackup-11 add-instance -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -D /var/lib/postgresql/11/main -INFO: Instance 'node' successfully inited - - - - Take a FULL backup: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b FULL --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YK2, backup mode: FULL, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YK2 -INFO: Backup PZ7YK2 data files are valid -INFO: Backup PZ7YK2 resident size: 196MB -INFO: Backup PZ7YK2 completed - - - - Let's take a look at the backup catalog: - -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' - -BACKUP INSTANCE 'pg-11' -================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -================================================================================================================================== - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - - - - Take an incremental backup in the DELTA mode: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YMP, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YK2 -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YMP -INFO: Backup PZ7YMP data files are valid -INFO: Backup PZ7YMP resident size: 32MB -INFO: Backup PZ7YMP completed - - - - Let's add some parameters to <application>pg_probackup</application> - configuration file, so that you can omit them from the command line: - -[backupman@backup_host] pg_probackup-11 set-config -B /mnt/backups --instance 'pg-11' --remote-host=postgres_host --remote-user=postgres -U backup -d backupdb - - - - Take another incremental backup in the DELTA mode, omitting - some of the previous parameters: - -[backupman@backup_host] pg_probackup-11 backup -B /mnt/backups --instance 'pg-11' -b delta --stream -INFO: Backup start, pg_probackup version: 2.2.0, instance: node, backup ID: PZ7YR5, backup mode: DELTA, wal mode: STREAM, remote: true, compress-algorithm: none, compress-level: 1 -INFO: Parent backup: PZ7YMP -INFO: Start transferring data files -INFO: Data files are transferred -INFO: wait for pg_stop_backup() -INFO: pg_stop backup() successfully executed -INFO: Validating backup PZ7YR5 -INFO: Backup PZ7YR5 data files are valid -INFO: Backup PZ7YR5 resident size: 32MB -INFO: Backup PZ7YR5 completed - - - - Let's take a look at the instance configuration: - -[backupman@backup_host] pg_probackup-11 show-config -B /mnt/backups --instance 'pg-11' - -# Backup instance information -pgdata = /var/lib/postgresql/11/main -system-identifier = 6746586934060931492 -xlog-seg-size = 16777216 -# Connection parameters -pgdatabase = backupdb -pghost = postgres_host -pguser = backup -# Replica parameters -replica-timeout = 5min -# Archive parameters -archive-timeout = 5min -# Logging parameters -log-level-console = INFO -log-level-file = OFF -log-filename = pg_probackup.log -log-rotation-size = 0 -log-rotation-age = 0 -# Retention parameters -retention-redundancy = 0 -retention-window = 0 -wal-depth = 0 -# Compression parameters -compress-algorithm = none -compress-level = 1 -# Remote access parameters -remote-proto = ssh -remote-host = postgres_host - - - Note that we are getting the default values for other options - that were not overwritten by the set-config command. - - - - Let's take a look at the backup catalog: - -[backupman@backup_host] pg_probackup-11 show -B /mnt/backups --instance 'pg-11' - -==================================================================================================================================== - Instance Version ID Recovery Time Mode WAL Mode TLI Time Data WAL Zratio Start LSN Stop LSN Status -==================================================================================================================================== - node 11 PZ7YR5 2019-10-11 19:49:56+03 DELTA STREAM 1/1 10s 112kB 32MB 1.00 0/41000028 0/41000160 OK - node 11 PZ7YMP 2019-10-11 19:47:16+03 DELTA STREAM 1/1 10s 376kB 32MB 1.00 0/3E000028 0/3F0000B8 OK - node 11 PZ7YK2 2019-10-11 19:45:45+03 FULL STREAM 1/0 11s 180MB 16MB 1.00 0/3C000028 0/3C000198 OK - - - - - - Versioning diff --git a/doc/stylesheet.css b/doc/stylesheet.css index 4d84058f5..31464154b 100644 --- a/doc/stylesheet.css +++ b/doc/stylesheet.css @@ -119,7 +119,8 @@ body { } .book code, kbd, pre, samp { - font-family: monospace,monospace; + font-family: monospace,monospace; + font-size: 90%; } .book .txtCommentsWrap { diff --git a/gen_probackup_project.pl b/gen_probackup_project.pl index b78f4699e..8143b7d0d 100644 --- a/gen_probackup_project.pl +++ b/gen_probackup_project.pl @@ -13,11 +13,11 @@ BEGIN { $pgsrc = shift @ARGV; if($pgsrc eq "--help"){ - print STDERR "Usage $0 pg-source-dir \n"; - print STDERR "Like this: \n"; - print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro \n"; - print STDERR "May be need input this before: \n"; - print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n"; + print STDERR "Usage $0 pg-source-dir\n"; + print STDERR "Like this:\n"; + print STDERR "$0 C:/PgProject/postgresql.10dev/postgrespro\n"; + print STDERR "May need to run this first:\n"; + print STDERR "CALL \"C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat\" amd64\n"; exit 1; } } @@ -133,7 +133,7 @@ sub build_pgprobackup unless (-d 'src/tools/msvc' && -d 'src'); # my $vsVersion = DetermineVisualStudioVersion(); - my $vsVersion = '12.00'; + my $vsVersion = '16.00'; $solution = CreateSolution($vsVersion, $config); @@ -155,6 +155,7 @@ sub build_pgprobackup 'archive.c', 'backup.c', 'catalog.c', + 'catchup.c', 'configure.c', 'data.c', 'delete.c', @@ -167,6 +168,7 @@ sub build_pgprobackup 'pg_probackup.c', 'restore.c', 'show.c', + 'stream.c', 'util.c', 'validate.c', 'checkdb.c', diff --git a/nls.mk b/nls.mk new file mode 100644 index 000000000..981c1c4fe --- /dev/null +++ b/nls.mk @@ -0,0 +1,6 @@ +# contrib/pg_probackup/nls.mk +CATALOG_NAME = pg_probackup +AVAIL_LANGUAGES = ru +GETTEXT_FILES = src/help.c +GETTEXT_TRIGGERS = $(FRONTEND_COMMON_GETTEXT_TRIGGERS) +GETTEXT_FLAGS = $(FRONTEND_COMMON_GETTEXT_FLAGS) diff --git a/packaging/Dockerfiles/Dockerfile-altlinux_8 b/packaging/Dockerfiles/Dockerfile-altlinux_8 new file mode 100644 index 000000000..961aa43dd --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-altlinux_8 @@ -0,0 +1,5 @@ +FROM alt:p8 +RUN ulimit -n 1024 && apt-get update -y && apt-get install -y tar wget rpm-build +RUN ulimit -n 1024 && apt-get install -y make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && apt-get install -y git perl-devel readline-devel libxml2-devel libxslt-devel python-devel zlib-devel openssl-devel libkrb5 libkrb5-devel +RUN ulimit -n 1024 && apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-altlinux_9 b/packaging/Dockerfiles/Dockerfile-altlinux_9 new file mode 100644 index 000000000..a75728720 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-altlinux_9 @@ -0,0 +1,5 @@ +FROM alt:p9 +RUN ulimit -n 1024 && apt-get update -y && apt-get install -y tar wget rpm-build +RUN ulimit -n 1024 && apt-get install -y make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && apt-get install -y git perl-devel readline-devel libxml2-devel libxslt-devel python-devel zlib-devel openssl-devel libkrb5 libkrb5-devel +RUN ulimit -n 1024 && apt-get dist-upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-astra_1.11 b/packaging/Dockerfiles/Dockerfile-astra_1.11 new file mode 100644 index 000000000..7db4999cd --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-astra_1.11 @@ -0,0 +1,7 @@ +FROM pgpro/astra:1.11 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install reprepro -y +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-centos_7 b/packaging/Dockerfiles/Dockerfile-centos_7 new file mode 100644 index 000000000..363440e85 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-centos_7 @@ -0,0 +1,5 @@ +FROM centos:7 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-centos_8 b/packaging/Dockerfiles/Dockerfile-centos_8 new file mode 100644 index 000000000..9de1d31b1 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-centos_8 @@ -0,0 +1,5 @@ +FROM centos:8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-createrepo1C b/packaging/Dockerfiles/Dockerfile-createrepo1C new file mode 100644 index 000000000..d987c4f5f --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-createrepo1C @@ -0,0 +1,4 @@ +FROM ubuntu:17.10 +RUN apt-get -qq update -y +RUN apt-get -qq install -y reprepro rpm createrepo gnupg rsync perl less wget expect rsync dpkg-dev +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_10 b/packaging/Dockerfiles/Dockerfile-debian_10 new file mode 100644 index 000000000..f25ceeac5 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_10 @@ -0,0 +1,7 @@ +FROM debian:10 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_11 b/packaging/Dockerfiles/Dockerfile-debian_11 new file mode 100644 index 000000000..db736c193 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_11 @@ -0,0 +1,7 @@ +FROM debian:11 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_8 b/packaging/Dockerfiles/Dockerfile-debian_8 new file mode 100644 index 000000000..0be9528bb --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_8 @@ -0,0 +1,7 @@ +FROM debian:8 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-debian_9 b/packaging/Dockerfiles/Dockerfile-debian_9 new file mode 100644 index 000000000..6ca10faa8 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-debian_9 @@ -0,0 +1,7 @@ +FROM debian:9 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_6 b/packaging/Dockerfiles/Dockerfile-oraclelinux_6 new file mode 100644 index 000000000..04325e869 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_6 @@ -0,0 +1,5 @@ +FROM oraclelinux:6 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_7 b/packaging/Dockerfiles/Dockerfile-oraclelinux_7 new file mode 100644 index 000000000..871d920eb --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_7 @@ -0,0 +1,5 @@ +FROM oraclelinux:7 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-oraclelinux_8 b/packaging/Dockerfiles/Dockerfile-oraclelinux_8 new file mode 100644 index 000000000..32e7cb03f --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-oraclelinux_8 @@ -0,0 +1,5 @@ +FROM oraclelinux:8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git openssl +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-rhel_7 b/packaging/Dockerfiles/Dockerfile-rhel_7 new file mode 100644 index 000000000..f64819e13 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rhel_7 @@ -0,0 +1,9 @@ +FROM registry.access.redhat.com/ubi7 +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/elfutils-0.176-5.el7.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/rpm-build-4.11.3-45.el7.x86_64.rpm +RUN yum install -y tar wget yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel +RUN yum install -y git +RUN yum upgrade -y +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/bison-3.0.4-2.el7.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/7/os/x86_64/Packages/flex-2.5.37-6.el7.x86_64.rpm diff --git a/packaging/Dockerfiles/Dockerfile-rhel_8 b/packaging/Dockerfiles/Dockerfile-rhel_8 new file mode 100644 index 000000000..82385785b --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rhel_8 @@ -0,0 +1,7 @@ +FROM registry.access.redhat.com/ubi8 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel +RUN yum install -y git +RUN yum upgrade -y +RUN yum install -y http://mirror.centos.org/centos/8/AppStream/x86_64/os/Packages/bison-3.0.4-10.el8.x86_64.rpm +RUN yum install -y http://mirror.centos.org/centos/8/AppStream/x86_64/os/Packages/flex-2.6.1-9.el8.x86_64.rpm diff --git a/packaging/Dockerfiles/Dockerfile-rosa_6 b/packaging/Dockerfiles/Dockerfile-rosa_6 new file mode 100644 index 000000000..42fa913e1 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-rosa_6 @@ -0,0 +1,5 @@ +FROM pgpro/rosa-6 +RUN yum install -y tar wget rpm-build yum-utils +RUN yum install -y gcc make perl libicu-devel glibc-devel bison flex +RUN yum install -y git +RUN yum upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-suse_15.1 b/packaging/Dockerfiles/Dockerfile-suse_15.1 new file mode 100644 index 000000000..afc9434a2 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-suse_15.1 @@ -0,0 +1,4 @@ +FROM opensuse/leap:15.1 +RUN ulimit -n 1024 && zypper install -y tar wget rpm-build +RUN ulimit -n 1024 && zypper install -y gcc make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && zypper install -y git rsync diff --git a/packaging/Dockerfiles/Dockerfile-suse_15.2 b/packaging/Dockerfiles/Dockerfile-suse_15.2 new file mode 100644 index 000000000..7e56e299a --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-suse_15.2 @@ -0,0 +1,4 @@ +FROM opensuse/leap:15.2 +RUN ulimit -n 1024 && zypper install -y tar wget rpm-build +RUN ulimit -n 1024 && zypper install -y gcc make perl libicu-devel glibc-devel bison flex +RUN ulimit -n 1024 && zypper install -y git rsync diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 new file mode 100644 index 000000000..10132f826 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_14.04 @@ -0,0 +1,7 @@ +FROM ubuntu:14.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 new file mode 100644 index 000000000..d511829c0 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_16.04 @@ -0,0 +1,7 @@ +FROM ubuntu:16.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 new file mode 100644 index 000000000..20a8567e0 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_18.04 @@ -0,0 +1,7 @@ +FROM ubuntu:18.04 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 b/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 new file mode 100644 index 000000000..66cefff16 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_18.10 @@ -0,0 +1,7 @@ +FROM ubuntu:18.10 +RUN apt-get update -y +RUN apt-get install -y devscripts +RUN apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN apt-get install -y cmake bison flex libboost-all-dev +RUN apt-get install -y reprepro +RUN apt-get upgrade -y diff --git a/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 b/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 new file mode 100644 index 000000000..79288d308 --- /dev/null +++ b/packaging/Dockerfiles/Dockerfile-ubuntu_20.04 @@ -0,0 +1,8 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND noninteractive +RUN ulimit -n 1024 && apt-get update -y +RUN ulimit -n 1024 && apt-get install -y devscripts +RUN ulimit -n 1024 && apt-get install -y dpkg-dev lsb-release git equivs wget vim +RUN ulimit -n 1024 && apt-get install -y cmake bison flex libboost-all-dev +RUN ulimit -n 1024 && apt-get install -y reprepro +RUN ulimit -n 1024 && apt-get upgrade -y diff --git a/packaging/Makefile.pkg b/packaging/Makefile.pkg new file mode 100644 index 000000000..e17243614 --- /dev/null +++ b/packaging/Makefile.pkg @@ -0,0 +1,185 @@ +ifeq ($(PBK_EDITION),std) + PBK_PKG_REPO = pg_probackup-forks + PBK_EDITION_FULL = Standard + PKG_NAME_SUFFIX = std- +else ifeq ($(PBK_EDITION),ent) + PBK_PKG_REPO = pg_probackup-forks + PBK_EDITION_FULL = Enterprise + PKG_NAME_SUFFIX = ent- +else + PBK_PKG_REPO = pg_probackup + PBK_EDITION_FULL = + PBK_EDITION = + PKG_NAME_SUFFIX = +endif + +check_env: + @if [ -z ${PBK_VERSION} ] ; then \ + echo "Env variable PBK_VERSION is not set" ; \ + false ; \ + fi + + @if [ -z ${PBK_RELEASE} ] ; then \ + echo "Env variable PBK_RELEASE is not set" ; \ + false ; \ + fi + + @if [ -z ${PBK_HASH} ] ; then \ + echo "Env variable PBK_HASH is not set" ; \ + false ; \ + fi + +pkg: check_env build/prepare build/all + @echo Build for all platform: done + +build/prepare: + mkdir -p build + +build/clean: build/prepare + find $(BUILDDIR) -maxdepth 1 -type f -exec rm -f {} \; + +build/all: build/debian build/ubuntu build/centos build/oraclelinux build/alt build/suse build/rhel + @echo Packaging is done + +### DEBIAN +build/debian: build/debian_9 build/debian_10 build/debian_11 + @echo Debian: done + +build/debian_9: build/debian_9_9.6 build/debian_9_10 build/debian_9_11 build/debian_9_12 build/debian_9_13 build/debian_9_14 + @echo Debian 9: done + +build/debian_10: build/debian_10_9.6 build/debian_10_10 build/debian_10_11 build/debian_10_12 build/debian_10_13 build/debian_10_14 + @echo Debian 10: done + +build/debian_11: build/debian_11_9.6 build/debian_11_10 build/debian_11_11 build/debian_11_12 build/debian_11_13 build/debian_11_14 + @echo Debian 11: done + +### UBUNTU +build/ubuntu: build/ubuntu_18.04 build/ubuntu_20.04 + @echo Ubuntu: done + +build/ubuntu_18.04: build/ubuntu_18.04_9.6 build/ubuntu_18.04_10 build/ubuntu_18.04_11 build/ubuntu_18.04_12 build/ubuntu_18.04_13 build/ubuntu_18.04_14 + @echo Ubuntu 18.04: done + +build/ubuntu_20.04: build/ubuntu_20.04_9.6 build/ubuntu_20.04_10 build/ubuntu_20.04_11 build/ubuntu_20.04_12 build/ubuntu_20.04_13 build/ubuntu_20.04_14 + @echo Ubuntu 20.04: done + +define build_deb + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg-probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg-probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/deb.sh +endef + +include packaging/pkg/Makefile.debian +include packaging/pkg/Makefile.ubuntu + +# CENTOS +build/centos: build/centos_7 build/centos_8 #build/rpm_repo_package_centos + @echo Centos: done + +build/centos_7: build/centos_7_9.6 build/centos_7_10 build/centos_7_11 build/centos_7_12 build/centos_7_13 build/centos_7_14 + @echo Centos 7: done + +# pgpro-9.6@centos-8 doesn't exist +build/centos_8: build/centos_8_10 build/centos_8_11 build/centos_8_12 build/centos_8_13 build/centos_8_14 #build/centos_8_9.6 + @echo Centos 8: done + +# Oracle Linux +build/oraclelinux: build/oraclelinux_6 build/oraclelinux_7 build/oraclelinux_8 #build/rpm_repo_package_oraclelinux + @echo Oraclelinux: done + +build/oraclelinux_6: build/oraclelinux_6_9.6 build/oraclelinux_6_10 build/oraclelinux_6_11 build/oraclelinux_6_12 build/oraclelinux_6_13 build/oraclelinux_6_14 + @echo Oraclelinux 6: done + +build/oraclelinux_7: build/oraclelinux_7_9.6 build/oraclelinux_7_10 build/oraclelinux_7_11 build/oraclelinux_7_12 build/oraclelinux_7_13 build/oraclelinux_7_14 + @echo Oraclelinux 7: done + +# pgpro-9.6@oraclelinux-8 doesn't exist +build/oraclelinux_8: build/oraclelinux_8_10 build/oraclelinux_8_11 build/oraclelinux_8_12 build/oraclelinux_8_13 build/oraclelinux_8_14 #build/oraclelinux_8_9.6 + @echo Oraclelinux 8: done + +# RHEL +build/rhel: build/rhel_7 build/rhel_8 #build/rpm_repo_package_rhel + @echo Rhel: done + +build/rhel_7: build/rhel_7_9.6 build/rhel_7_10 build/rhel_7_11 build/rhel_7_12 build/rhel_7_13 build/rhel_7_14 + @echo Rhel 7: done + +build/rhel_8: build/rhel_8_9.6 build/rhel_8_10 build/rhel_8_11 build/rhel_8_12 build/rhel_8_13 build/rhel_8_14 + @echo Rhel 8: done + + +define build_rpm + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/rpm.sh +endef + +include packaging/pkg/Makefile.centos +include packaging/pkg/Makefile.rhel +include packaging/pkg/Makefile.oraclelinux + + +# Alt Linux +build/alt: build/alt_7 build/alt_8 build/alt_9 + @echo Alt Linux: done + +build/alt_7: build/alt_7_9.6 build/alt_7_10 build/alt_7_11 build/alt_7_12 build/alt_7_13 build/alt_7_14 + @echo Alt Linux 7: done + +build/alt_8: build/alt_8_9.6 build/alt_8_10 build/alt_8_11 build/alt_8_12 build/alt_8_13 build/alt_8_14 + @echo Alt Linux 8: done + +build/alt_9: build/alt_9_9.6 build/alt_9_10 build/alt_9_11 build/alt_9_12 build/alt_9_13 build/alt_9_14 + @echo Alt Linux 9: done + +define build_alt + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/alt.sh +endef + +include packaging/pkg/Makefile.alt + +# SUSE Linux +build/suse: build/suse_15.1 build/suse_15.2 + @echo Suse: done + +# there is no PG-14 in suse-15.1 repositories (test fails) +build/suse_15.1: build/suse_15.1_9.6 build/suse_15.1_10 build/suse_15.1_11 build/suse_15.1_12 build/suse_15.1_13 + @echo Rhel 15.1: done + +build/suse_15.2: build/suse_15.2_9.6 build/suse_15.2_10 build/suse_15.2_11 build/suse_15.2_12 build/suse_15.2_13 build/suse_15.2_14 + @echo Rhel 15.1: done + +define build_suse + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/pkg:/app/in \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2/pg_probackup-$(PKG_NAME_SUFFIX)$4/$(PBK_VERSION):/app/out \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/suse.sh +endef + +include packaging/pkg/Makefile.suse diff --git a/packaging/Makefile.repo b/packaging/Makefile.repo new file mode 100644 index 000000000..10fb27137 --- /dev/null +++ b/packaging/Makefile.repo @@ -0,0 +1,156 @@ +#### REPO BUILD #### +repo: check_env repo/debian repo/ubuntu repo/centos repo/oraclelinux repo/rhel repo/alt repo/suse repo_finish + @echo Build repo for all platform: done + +# Debian +repo/debian: build/repo_debian_9 build/repo_debian_10 build/repo_debian_11 + @echo Build repo for debian platforms: done + +build/repo_debian_9: + $(call build_repo_deb,debian,9,stretch) + touch build/repo_debian_9 + +build/repo_debian_10: + $(call build_repo_deb,debian,10,buster) + touch build/repo_debian_10 + +build/repo_debian_11: + $(call build_repo_deb,debian,11,bullseye) + touch build/repo_debian_11 + +# Ubuntu +repo/ubuntu: build/repo_ubuntu_18.04 build/repo_ubuntu_20.04 + @echo Build repo for ubuntu platforms: done + +build/repo_ubuntu_18.04: + $(call build_repo_deb,ubuntu,18.04,bionic) + touch build/repo_ubuntu_18.04 + +build/repo_ubuntu_20.04: + $(call build_repo_deb,ubuntu,20.04,focal) + touch build/repo_ubuntu_20.04 + +# Centos +repo/centos: build/repo_centos_7 build/repo_centos_8 + @echo Build repo for centos platforms: done + +build/repo_centos_7: + $(call build_repo_rpm,centos,7,,) + touch build/repo_centos_7 + +build/repo_centos_8: + $(call build_repo_rpm,centos,8,,) + touch build/repo_centos_8 + +# Oraclelinux +repo/oraclelinux: build/repo_oraclelinux_6 build/repo_oraclelinux_7 build/repo_oraclelinux_8 + @echo Build repo for oraclelinux platforms: done + +build/repo_oraclelinux_6: + $(call build_repo_rpm,oraclelinux,6,6Server) + touch build/repo_oraclelinux_6 + +build/repo_oraclelinux_7: + $(call build_repo_rpm,oraclelinux,7,7Server) + touch build/repo_oraclelinux_7 + +build/repo_oraclelinux_8: + $(call build_repo_rpm,oraclelinux,8,,) + touch build/repo_oraclelinux_8 + +# RHEL +repo/rhel: build/repo_rhel_7 build/repo_rhel_8 + @echo Build repo for rhel platforms: done + +build/repo_rhel_7: + $(call build_repo_rpm,rhel,7,7Server) + touch build/repo_rhel_7 + +build/repo_rhel_8: + $(call build_repo_rpm,rhel,8,,) + touch build/repo_rhel_8 + +# ALT +repo/alt: build/repo_alt_7 build/repo_alt_8 build/repo_alt_9 + @echo Build repo for alt platforms: done + +build/repo_alt_7: + $(call build_repo_alt,alt,7,,) + touch build/repo_alt_7 + +build/repo_alt_8: + $(call build_repo_alt,alt,8,,) + touch build/repo_alt_8 + +build/repo_alt_9: + $(call build_repo_alt,alt,9,,) + touch build/repo_alt_9 + +# SUSE +repo/suse: build/repo_suse_15.1 build/repo_suse_15.2 + @echo Build repo for suse platforms: done + +build/repo_suse_15.1: + $(call build_repo_suse,suse,15.1,,) + touch build/repo_suse_15.1 + +build/repo_suse_15.2: + $(call build_repo_suse,suse,15.2,,) + touch build/repo_suse_15.2 + +repo_finish: +# cd build/data/www/$(PBK_PKG_REPO)/ + cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/rpm && sudo ln -nsf $(PBK_VERSION) latest + # following line only for vanilla + cd $(BUILDDIR)/data/www/$(PBK_PKG_REPO)/srpm && sudo ln -nsf $(PBK_VERSION) latest + +# sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/rpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/rpm/latest +# sudo ln -rfs build/data/www/$(PBK_PKG_REPO)/srpm/${PBK_VERSION} build/data/www/$(PBK_PKG_REPO)/srpm/latest + +define build_repo_deb + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/repo /app/repo/scripts/deb.sh +endef + +define build_repo_rpm + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/repo /app/repo/scripts/rpm.sh +endef + +define build_repo_alt + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/$1:$2 /app/repo/scripts/alt.sh +endef + +define build_repo_suse + docker rm -f $1_$2_pbk_repo >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/repo:/app/repo \ + -v $(WORKDIR)/build/data/www:/app/www \ + -v $(WORKDIR)/build/data/$(PBK_PKG_REPO)/$1/$2:/app/in \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" \ + -e "PBK_PKG_REPO=$(PBK_PKG_REPO)" -e "PBK_EDITION=$(PBK_EDITION)" \ + --name $1_$2_pbk_repo \ + --rm pgpro/$1:$2 /app/repo/scripts/suse.sh +endef diff --git a/packaging/Makefile.test b/packaging/Makefile.test new file mode 100644 index 000000000..11c63619a --- /dev/null +++ b/packaging/Makefile.test @@ -0,0 +1,150 @@ +ifeq ($(PBK_EDITION),std) + SCRIPT_SUFFIX = _forks +else ifeq ($(PBK_EDITION),ent) + SCRIPT_SUFFIX = _forks +else + SCRIPT_SUFFIX = +endif + +test: build/test_all + @echo Test for all platform: done + +build/test_all: build/test_debian build/test_ubuntu build/test_centos build/test_oraclelinux build/test_alt build/test_suse #build/test_rhel + @echo Package testing is done + +### DEBIAN +build/test_debian: build/test_debian_9 build/test_debian_10 build/test_debian_11 + @echo Debian: done + +build/test_debian_9: build/test_debian_9_9.6 build/test_debian_9_10 build/test_debian_9_11 build/test_debian_9_12 build/test_debian_9_13 build/test_debian_9_14 + @echo Debian 9: done + +build/test_debian_10: build/test_debian_10_9.6 build/test_debian_10_10 build/test_debian_10_11 build/test_debian_10_12 build/test_debian_10_13 build/test_debian_10_14 + @echo Debian 10: done + +build/test_debian_11: build/test_debian_11_9.6 build/test_debian_11_10 build/test_debian_11_11 build/test_debian_11_12 build/test_debian_11_13 build/test_debian_11_14 + @echo Debian 11: done + +### UBUNTU +build/test_ubuntu: build/test_ubuntu_18.04 build/test_ubuntu_20.04 + @echo Ubuntu: done + +build/test_ubuntu_18.04: build/test_ubuntu_18.04_9.6 build/test_ubuntu_18.04_10 build/test_ubuntu_18.04_11 build/test_ubuntu_18.04_12 build/test_ubuntu_18.04_13 build/test_ubuntu_18.04_14 + @echo Ubuntu 18.04: done + +build/test_ubuntu_20.04: build/test_ubuntu_20.04_9.6 build/test_ubuntu_20.04_10 build/test_ubuntu_20.04_11 build/test_ubuntu_20.04_12 build/test_ubuntu_20.04_13 build/test_ubuntu_20.04_14 + @echo Ubuntu 20.04: done + +define test_deb + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg-probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/deb$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.debian +include packaging/test/Makefile.ubuntu + +# CENTOS +build/test_centos: build/test_centos_7 build/test_centos_8 + @echo Centos: done + +build/test_centos_7: build/test_centos_7_9.6 build/test_centos_7_10 build/test_centos_7_11 build/test_centos_7_12 build/test_centos_7_13 #build/test_centos_7_14 + @echo Centos 7: done + +# pgpro-9.6@centos-8 doesn't exist +build/test_centos_8: build/test_centos_8_10 build/test_centos_8_11 build/test_centos_8_12 build/test_centos_8_13 #build/test_centos_8_14 build/test_centos_8_9.6 + @echo Centos 8: done + +# Oracle Linux +build/test_oraclelinux: build/test_oraclelinux_7 build/test_oraclelinux_8 + @echo Oraclelinux: done + +build/test_oraclelinux_7: build/test_oraclelinux_7_9.6 build/test_oraclelinux_7_10 build/test_oraclelinux_7_11 build/test_oraclelinux_7_12 build/test_oraclelinux_7_13 #build/test_oraclelinux_7_14 + @echo Oraclelinux 7: done + +# pgpro-9.6@oraclelinux-8 doesn't exist +build/test_oraclelinux_8: build/test_oraclelinux_8_10 build/test_oraclelinux_8_11 build/test_oraclelinux_8_12 build/test_oraclelinux_8_13 #build/test_oraclelinux_8_14 build/test_oraclelinux_8_9.6 + @echo Oraclelinux 8: done + +# RHEL +build/test_rhel: build/test_rhel_7 #build/test_rhel_8 + @echo Rhel: done + +build/test_rhel_7: build/test_rhel_7_9.6 build/test_rhel_7_10 build/test_rhel_7_11 build/test_rhel_7_12 build/test_rhel_7_13 #build/test_rhel_7_14 + @echo Rhel 7: done + +build/test_rhel_8: build/test_rhel_8_9.6 build/test_rhel_8_10 build/test_rhel_8_11 build/test_rhel_8_12 build/test_rhel_8_13 build/test_rhel_8_14 + @echo Rhel 8: done + +define test_rpm + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/rpm$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.centos +include packaging/test/Makefile.rhel +include packaging/test/Makefile.oraclelinux + +# Alt Linux +build/test_alt: build/test_alt_8 build/test_alt_9 + @echo Alt Linux: done + +# nginx@alt7 fall with 'nginx: [alert] sysctl(KERN_RTSIGMAX) failed (1: Operation not permitted)' +# within docker on modern host linux kernels (this nginx build require Linux between 2.2.19 and 2.6.17) + +build/test_alt_8: build/test_alt_8_9.6 build/test_alt_8_10 build/test_alt_8_11 build/test_alt_8_12 build/test_alt_8_13 build/test_alt_8_14 + @echo Alt Linux 8: done + +build/test_alt_9: build/test_alt_9_9.6 build/test_alt_9_10 build/test_alt_9_11 build/test_alt_9_12 build/test_alt_9_13 build/test_alt_9_14 + @echo Alt Linux 9: done + +define test_alt + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/alt$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.alt + +# SUSE Linux +build/test_suse: build/test_suse_15.1 build/test_suse_15.2 + @echo Suse: done + +build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13 + @echo Suse 15.1: done + +build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14 + @echo Suse 15.2: done + +define test_suse + docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \ + docker run \ + -v $(WORKDIR)/packaging/test:/app/in \ + -v $(BUILDDIR)/data/www:/app/www \ + -e "DISTRIB=$1" -e "DISTRIB_VERSION=$2" -e "CODENAME=$3" -e "PG_VERSION=$4" -e "PG_FULL_VERSION=$5" \ + -e "PKG_HASH=$(PBK_HASH)" -e "PKG_URL=$(PBK_GIT_REPO)" -e "PKG_RELEASE=$(PBK_RELEASE)" -e "PKG_NAME=pg_probackup-$(PKG_NAME_SUFFIX)$4" \ + -e "PKG_VERSION=$(PBK_VERSION)" -e "PBK_EDITION=$(PBK_EDITION)" -e "PBK_EDITION_FULL=$(PBK_EDITION_FULL)" \ + --name $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION)_pg_$5 \ + --rm pgpro/$1:$2 /app/in/scripts/suse$(SCRIPT_SUFFIX).sh +endef + +include packaging/test/Makefile.suse diff --git a/packaging/Readme.md b/packaging/Readme.md new file mode 100644 index 000000000..f4437d838 --- /dev/null +++ b/packaging/Readme.md @@ -0,0 +1,23 @@ +Example: +``` +export PBK_VERSION=2.4.17 +export PBK_HASH=57f871accce2604 +export PBK_RELEASE=1 +export PBK_EDITION=std|ent +make --keep-going pkg +``` + +To build binaries for PostgresPro Standard or Enterprise, a pgpro.tar.bz2 with latest git tree must be preset in `packaging/pkg/tarballs` directory: +``` +cd packaging/pkg/tarballs +git clone pgpro_repo pgpro +tar -cjSf pgpro.tar.bz2 pgpro +``` + +To build repo the gpg keys for package signing must be present ... +Repo must be build using 1 thread (due to debian bullshit): +``` +make repo -j1 +``` + + diff --git a/packaging/pkg/Makefile.alt b/packaging/pkg/Makefile.alt new file mode 100644 index 000000000..28eabf53f --- /dev/null +++ b/packaging/pkg/Makefile.alt @@ -0,0 +1,87 @@ +# ALT 7 +build/alt_7_9.5: + $(call build_alt,alt,7,,9.5,9.5.25) + touch build/alt_7_9.5 + +build/alt_7_9.6: + $(call build_alt,alt,7,,9.6,9.6.24) + touch build/alt_7_9.6 + +build/alt_7_10: + $(call build_alt,alt,7,,10,10.19) + touch build/alt_7_10 + +build/alt_7_11: + $(call build_alt,alt,7,,11,11.14) + touch build/alt_7_11 + +build/alt_7_12: + $(call build_alt,alt,7,,12,12.9) + touch build/alt_7_12 + +build/alt_7_13: + $(call build_alt,alt,7,,13,13.5) + touch build/alt_7_13 + +build/alt_7_14: + $(call build_alt,alt,7,,14,14.1) + touch build/alt_7_14 + +# ALT 8 +build/alt_8_9.5: + $(call build_alt,alt,8,,9.5,9.5.25) + touch build/alt_8_9.5 + +build/alt_8_9.6: + $(call build_alt,alt,8,,9.6,9.6.24) + touch build/alt_8_9.6 + +build/alt_8_10: + $(call build_alt,alt,8,,10,10.19) + touch build/alt_8_10 + +build/alt_8_11: + $(call build_alt,alt,8,,11,11.14) + touch build/alt_8_11 + +build/alt_8_12: + $(call build_alt,alt,8,,12,12.9) + touch build/alt_8_12 + +build/alt_8_13: + $(call build_alt,alt,8,,13,13.5) + touch build/alt_8_13 + +build/alt_8_14: + $(call build_alt,alt,8,,14,14.1) + touch build/alt_8_14 + +# ALT 9 +build/alt_9_9.5: + $(call build_alt,alt,9,,9.5,9.5.25) + touch build/alt_9_9.5 + +build/alt_9_9.6: + $(call build_alt,alt,9,,9.6,9.6.24) + touch build/alt_9_9.6 + +build/alt_9_10: + $(call build_alt,alt,9,,10,10.19) + touch build/alt_9_10 + +build/alt_9_11: + $(call build_alt,alt,9,,11,11.14) + touch build/alt_9_11 + +build/alt_9_12: + $(call build_alt,alt,9,,12,12.9) + touch build/alt_9_12 + +build/alt_9_13: + $(call build_alt,alt,9,,13,13.5) + touch build/alt_9_13 + +build/alt_9_14: + $(call build_alt,alt,9,,14,14.1) + touch build/alt_9_14 + diff --git a/packaging/pkg/Makefile.centos b/packaging/pkg/Makefile.centos new file mode 100644 index 000000000..fb537d0a6 --- /dev/null +++ b/packaging/pkg/Makefile.centos @@ -0,0 +1,57 @@ +# CENTOS 7 +build/centos_7_9.5: + $(call build_rpm,centos,7,,9.5,9.5.25) + touch build/centos_7_9.5 + +build/centos_7_9.6: + $(call build_rpm,centos,7,,9.6,9.6.24) + touch build/centos_7_9.6 + +build/centos_7_10: + $(call build_rpm,centos,7,,10,10.19) + touch build/centos_7_10 + +build/centos_7_11: + $(call build_rpm,centos,7,,11,11.14) + touch build/centos_7_11 + +build/centos_7_12: + $(call build_rpm,centos,7,,12,12.9) + touch build/centos_7_12 + +build/centos_7_13: + $(call build_rpm,centos,7,,13,13.5) + touch build/centos_7_13 + +build/centos_7_14: + $(call build_rpm,centos,7,,14,14.1) + touch build/centos_7_14 + +# CENTOS 8 +build/centos_8_9.5: + $(call build_rpm,centos,8,,9.5,9.5.25) + touch build/centos_8_9.5 + +build/centos_8_9.6: + $(call build_rpm,centos,8,,9.6,9.6.24) + touch build/centos_8_9.6 + +build/centos_8_10: + $(call build_rpm,centos,8,,10,10.19) + touch build/centos_8_10 + +build/centos_8_11: + $(call build_rpm,centos,8,,11,11.14) + touch build/centos_8_11 + +build/centos_8_12: + $(call build_rpm,centos,8,,12,12.9) + touch build/centos_8_12 + +build/centos_8_13: + $(call build_rpm,centos,8,,13,13.5) + touch build/centos_8_13 + +build/centos_8_14: + $(call build_rpm,centos,8,,14,14.1) + touch build/centos_8_14 diff --git a/packaging/pkg/Makefile.debian b/packaging/pkg/Makefile.debian new file mode 100644 index 000000000..d9c885d3a --- /dev/null +++ b/packaging/pkg/Makefile.debian @@ -0,0 +1,86 @@ +# DEBIAN 9 +build/debian_9_9.5: + $(call build_deb,debian,9,stretch,9.5,9.5.25) + touch build/debian_9_9.5 + +build/debian_9_9.6: + $(call build_deb,debian,9,stretch,9.6,9.6.24) + touch build/debian_9_9.6 + +build/debian_9_10: + $(call build_deb,debian,9,stretch,10,10.19) + touch build/debian_9_10 + +build/debian_9_11: + $(call build_deb,debian,9,stretch,11,11.14) + touch build/debian_9_11 + +build/debian_9_12: + $(call build_deb,debian,9,stretch,12,12.9) + touch build/debian_9_12 + +build/debian_9_13: + $(call build_deb,debian,9,stretch,13,13.5) + touch build/debian_9_13 + +build/debian_9_14: + $(call build_deb,debian,9,stretch,14,14.1) + touch build/debian_9_14 + +# DEBIAN 10 +build/debian_10_9.5: + $(call build_deb,debian,10,buster,9.5,9.5.25) + touch build/debian_10_9.5 + +build/debian_10_9.6: + $(call build_deb,debian,10,buster,9.6,9.6.24) + touch build/debian_10_9.6 + +build/debian_10_10: + $(call build_deb,debian,10,buster,10,10.19) + touch build/debian_10_10 + +build/debian_10_11: + $(call build_deb,debian,10,buster,11,11.14) + touch build/debian_10_11 + +build/debian_10_12: + $(call build_deb,debian,10,buster,12,12.9) + touch build/debian_10_12 + +build/debian_10_13: + $(call build_deb,debian,10,buster,13,13.5) + touch build/debian_10_13 + +build/debian_10_14: + $(call build_deb,debian,10,buster,14,14.1) + touch build/debian_10_14 + +# DEBIAN 11 +build/debian_11_9.5: + $(call build_deb,debian,11,bullseye,9.5,9.5.25) + touch build/debian_11_9.5 + +build/debian_11_9.6: + $(call build_deb,debian,11,bullseye,9.6,9.6.24) + touch build/debian_11_9.6 + +build/debian_11_10: + $(call build_deb,debian,11,bullseye,10,10.19) + touch build/debian_11_10 + +build/debian_11_11: + $(call build_deb,debian,11,bullseye,11,11.14) + touch build/debian_11_11 + +build/debian_11_12: + $(call build_deb,debian,11,bullseye,12,12.9) + touch build/debian_11_12 + +build/debian_11_13: + $(call build_deb,debian,11,bullseye,13,13.5) + touch build/debian_11_13 + +build/debian_11_14: + $(call build_deb,debian,11,bullseye,14,14.1) + touch build/debian_11_14 diff --git a/packaging/pkg/Makefile.oraclelinux b/packaging/pkg/Makefile.oraclelinux new file mode 100644 index 000000000..127a578f1 --- /dev/null +++ b/packaging/pkg/Makefile.oraclelinux @@ -0,0 +1,87 @@ +# ORACLE LINUX 6 +build/oraclelinux_6_9.5: + $(call build_rpm,oraclelinux,6,,9.5,9.5.25) + touch build/oraclelinux_6_9.5 + +build/oraclelinux_6_9.6: + $(call build_rpm,oraclelinux,6,,9.6,9.6.24) + touch build/oraclelinux_6_9.6 + +build/oraclelinux_6_10: + $(call build_rpm,oraclelinux,6,,10,10.19) + touch build/oraclelinux_6_10 + +build/oraclelinux_6_11: + $(call build_rpm,oraclelinux,6,,11,11.14) + touch build/oraclelinux_6_11 + +build/oraclelinux_6_12: + $(call build_rpm,oraclelinux,6,,12,12.9) + touch build/oraclelinux_6_12 + +build/oraclelinux_6_13: + $(call build_rpm,oraclelinux,6,,13,13.5) + touch build/oraclelinux_6_13 + +build/oraclelinux_6_14: + $(call build_rpm,oraclelinux,6,,14,14.1) + touch build/oraclelinux_6_14 + +# ORACLE LINUX 7 +build/oraclelinux_7_9.5: + $(call build_rpm,oraclelinux,7,,9.5,9.5.25) + touch build/oraclelinux_7_9.5 + +build/oraclelinux_7_9.6: + $(call build_rpm,oraclelinux,7,,9.6,9.6.24) + touch build/oraclelinux_7_9.6 + +build/oraclelinux_7_10: + $(call build_rpm,oraclelinux,7,,10,10.19) + touch build/oraclelinux_7_10 + +build/oraclelinux_7_11: + $(call build_rpm,oraclelinux,7,,11,11.14) + touch build/oraclelinux_7_11 + +build/oraclelinux_7_12: + $(call build_rpm,oraclelinux,7,,12,12.9) + touch build/oraclelinux_7_12 + +build/oraclelinux_7_13: + $(call build_rpm,oraclelinux,7,,13,13.5) + touch build/oraclelinux_7_13 + +build/oraclelinux_7_14: + $(call build_rpm,oraclelinux,7,,14,14.1) + touch build/oraclelinux_7_14 + +# ORACLE LINUX 8 +build/oraclelinux_8_9.5: + $(call build_rpm,oraclelinux,8,,9.5,9.5.25) + touch build/oraclelinux_8_9.5 + +build/oraclelinux_8_9.6: + $(call build_rpm,oraclelinux,8,,9.6,9.6.24) + touch build/oraclelinux_8_9.6 + +build/oraclelinux_8_10: + $(call build_rpm,oraclelinux,8,,10,10.19) + touch build/oraclelinux_8_10 + +build/oraclelinux_8_11: + $(call build_rpm,oraclelinux,8,,11,11.14) + touch build/oraclelinux_8_11 + +build/oraclelinux_8_12: + $(call build_rpm,oraclelinux,8,,12,12.9) + touch build/oraclelinux_8_12 + +build/oraclelinux_8_13: + $(call build_rpm,oraclelinux,8,,13,13.5) + touch build/oraclelinux_8_13 + +build/oraclelinux_8_14: + $(call build_rpm,oraclelinux,8,,14,14.1) + touch build/oraclelinux_8_14 + diff --git a/packaging/pkg/Makefile.rhel b/packaging/pkg/Makefile.rhel new file mode 100644 index 000000000..8c1b0687b --- /dev/null +++ b/packaging/pkg/Makefile.rhel @@ -0,0 +1,57 @@ +# RHEL 7 +build/rhel_7_9.5: + $(call build_rpm,rhel,7,7Server,9.5,9.5.25) + touch build/rhel_7_9.5 + +build/rhel_7_9.6: + $(call build_rpm,rhel,7,7Server,9.6,9.6.24) + touch build/rhel_7_9.6 + +build/rhel_7_10: + $(call build_rpm,rhel,7,7Server,10,10.19) + touch build/rhel_7_10 + +build/rhel_7_11: + $(call build_rpm,rhel,7,7Server,11,11.14) + touch build/rhel_7_11 + +build/rhel_7_12: + $(call build_rpm,rhel,7,7Server,12,12.9) + touch build/rhel_7_12 + +build/rhel_7_13: + $(call build_rpm,rhel,7,7Server,13,13.5) + touch build/rhel_7_13 + +build/rhel_7_14: + $(call build_rpm,rhel,7,7Server,14,14.1) + touch build/rhel_7_14 + +# RHEL 8 +build/rhel_8_9.5: + $(call build_rpm,rhel,8,8Server,9.5,9.5.25) + touch build/rhel_8_9.5 + +build/rhel_8_9.6: + $(call build_rpm,rhel,8,8Server,9.6,9.6.24) + touch build/rhel_8_9.6 + +build/rhel_8_10: + $(call build_rpm,rhel,8,8Server,10,10.19) + touch build/rhel_8_10 + +build/rhel_8_11: + $(call build_rpm,rhel,8,8Server,11,11.14) + touch build/rhel_8_11 + +build/rhel_8_12: + $(call build_rpm,rhel,8,8Server,12,12.9) + touch build/rhel_8_12 + +build/rhel_8_13: + $(call build_rpm,rhel,8,8Server,13,13.5) + touch build/rhel_8_13 + +build/rhel_8_14: + $(call build_rpm,rhel,8,8Server,14,14.1) + touch build/rhel_8_14 diff --git a/packaging/pkg/Makefile.suse b/packaging/pkg/Makefile.suse new file mode 100644 index 000000000..c71ebd389 --- /dev/null +++ b/packaging/pkg/Makefile.suse @@ -0,0 +1,57 @@ +# Suse 15.1 +build/suse_15.1_9.5: + $(call build_suse,suse,15.1,,9.5,9.5.25) + touch build/suse_15.1_9.5 + +build/suse_15.1_9.6: + $(call build_suse,suse,15.1,,9.6,9.6.24) + touch build/suse_15.1_9.6 + +build/suse_15.1_10: + $(call build_suse,suse,15.1,,10,10.19) + touch build/suse_15.1_10 + +build/suse_15.1_11: + $(call build_suse,suse,15.1,,11,11.14) + touch build/suse_15.1_11 + +build/suse_15.1_12: + $(call build_suse,suse,15.1,,12,12.9) + touch build/suse_15.1_12 + +build/suse_15.1_13: + $(call build_suse,suse,15.1,,13,13.5) + touch build/suse_15.1_13 + +build/suse_15.1_14: + $(call build_suse,suse,15.1,,14,14.1) + touch build/suse_15.1_14 + +# Suse 15.2 +build/suse_15.2_9.5: + $(call build_suse,suse,15.2,,9.5,9.5.25) + touch build/suse_15.2_9.5 + +build/suse_15.2_9.6: + $(call build_suse,suse,15.2,,9.6,9.6.24) + touch build/suse_15.2_9.6 + +build/suse_15.2_10: + $(call build_suse,suse,15.2,,10,10.19) + touch build/suse_15.2_10 + +build/suse_15.2_11: + $(call build_suse,suse,15.2,,11,11.14) + touch build/suse_15.2_11 + +build/suse_15.2_12: + $(call build_suse,suse,15.2,,12,12.9) + touch build/suse_15.2_12 + +build/suse_15.2_13: + $(call build_suse,suse,15.2,,13,13.5) + touch build/suse_15.2_13 + +build/suse_15.2_14: + $(call build_suse,suse,15.2,,14,14.1) + touch build/suse_15.2_14 diff --git a/packaging/pkg/Makefile.ubuntu b/packaging/pkg/Makefile.ubuntu new file mode 100644 index 000000000..02acd6c67 --- /dev/null +++ b/packaging/pkg/Makefile.ubuntu @@ -0,0 +1,58 @@ +# UBUNTU 20.04 +build/ubuntu_20.04_9.5: + $(call build_deb,ubuntu,20.04,focal,9.5,9.5.25) + touch build/ubuntu_20.04_9.5 + +build/ubuntu_20.04_9.6: + $(call build_deb,ubuntu,20.04,focal,9.6,9.6.24) + touch build/ubuntu_20.04_9.6 + +build/ubuntu_20.04_10: + $(call build_deb,ubuntu,20.04,focal,10,10.19) + touch build/ubuntu_20.04_10 + +build/ubuntu_20.04_11: + $(call build_deb,ubuntu,20.04,focal,11,11.14) + touch build/ubuntu_20.04_11 + +build/ubuntu_20.04_12: + $(call build_deb,ubuntu,20.04,focal,12,12.9) + touch build/ubuntu_20.04_12 + +build/ubuntu_20.04_13: + $(call build_deb,ubuntu,20.04,focal,13,13.5) + touch build/ubuntu_20.04_13 + +build/ubuntu_20.04_14: + $(call build_deb,ubuntu,20.04,focal,14,14.1) + touch build/ubuntu_20.04_14 + +# UBUNTU 18.04 +build/ubuntu_18.04_9.5: + $(call build_deb,ubuntu,18.04,bionic,9.5,9.5.25) + touch build/ubuntu_18.04_9.5 + +build/ubuntu_18.04_9.6: + $(call build_deb,ubuntu,18.04,bionic,9.6,9.6.24) + touch build/ubuntu_18.04_9.6 + +build/ubuntu_18.04_10: + $(call build_deb,ubuntu,18.04,bionic,10,10.19) + touch build/ubuntu_18.04_10 + +build/ubuntu_18.04_11: + $(call build_deb,ubuntu,18.04,bionic,11,11.14) + touch build/ubuntu_18.04_11 + +build/ubuntu_18.04_12: + $(call build_deb,ubuntu,18.04,bionic,12,12.9) + touch build/ubuntu_18.04_12 + +build/ubuntu_18.04_13: + $(call build_deb,ubuntu,18.04,bionic,13,13.5) + touch build/ubuntu_18.04_13 + +build/ubuntu_18.04_14: + $(call build_deb,ubuntu,18.04,bionic,14,14.1) + touch build/ubuntu_18.04_14 + diff --git a/packaging/pkg/scripts/alt.sh b/packaging/pkg/scripts/alt.sh new file mode 100755 index 000000000..7c3971d6a --- /dev/null +++ b/packaging/pkg/scripts/alt.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# THere is no std/ent packages for PG 9.5 +if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 +apt-get update -y + +mkdir /root/build +cd /root/build + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +# tarball it +if [[ ${PBK_EDITION} == '' ]] ; then + tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} + mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES + rm -rf pg_probackup-${PKG_VERSION} +else + mv pg_probackup-${PKG_VERSION} /root/rpmbuild/SOURCES +fi + + +if [[ ${PBK_EDITION} == '' ]] ; then + # Download PostgreSQL source + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O postgresql-${PG_VERSION}.tar.bz2 + mv postgresql-${PG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES/ + +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/rpmbuild/SOURCES/ + cd /root/rpmbuild/SOURCES/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + rm -rf .git + + cd /root/rpmbuild/SOURCES/ + mv pgpro postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + chown -R root:root postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + + +#cd /root/rpmbuild/SOURCES +#sed -i "s/@PG_VERSION@/${PKG_VERSION}/" pg_probackup.repo + +# build postgresql +echo '%_allow_root_build yes' > /root/.rpmmacros +echo '%_topdir %{getenv:HOME}/rpmbuild' >> /root/.rpmmacros + +cd /root/rpmbuild/SPECS +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.alt.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.alt.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.alt.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.alt.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.alt.spec +else + sed -i "s/@EDITION@/${PBK_EDITION}/" pg_probackup.alt.forks.spec + sed -i "s/@EDITION_FULL@/${PBK_EDITION_FULL}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.alt.forks.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.alt.forks.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.alt.forks.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.alt.forks.spec + + if [ ${PG_VERSION} != '9.6' ]; then + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup.alt.forks.spec + fi +fi + +# ALT Linux suck as detecting dependecies, so the manual hint is required +if [ ${DISTRIB_VERSION} == '7' ]; then + apt-get install libpq5.10 + +elif [ ${DISTRIB_VERSION} == '8' ]; then + apt-get install libpq5.12 + +else + apt-get install libpq5 +fi + +# install dependencies +#stolen from postgrespro +apt-get install -y flex libldap-devel libpam-devel libreadline-devel libssl-devel + +if [[ ${PBK_EDITION} == '' ]] ; then + # build pg_probackup + rpmbuild -bs pg_probackup.alt.spec + rpmbuild -ba pg_probackup.alt.spec #2>&1 | tee -ai /app/out/build.log + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out +else + rpmbuild -ba pg_probackup.alt.forks.spec #2>&1 | tee -ai /app/out/build.log + # write artefacts to out directory + rm -rf /app/out/* + # cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out + cp -arv /root/rpmbuild/RPMS /app/out +fi diff --git a/packaging/pkg/scripts/deb.sh b/packaging/pkg/scripts/deb.sh new file mode 100755 index 000000000..6e134635a --- /dev/null +++ b/packaging/pkg/scripts/deb.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# There is no std/ent packages for PG 9.5 +if [[ ${PG_VERSION} == '9.5' ]] && [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# PACKAGES NEEDED +apt-get --allow-releaseinfo-change update -y && apt-get install -y git wget bzip2 devscripts equivs + +# Prepare +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +if [ ${CODENAME} == 'jessie' ]; then + printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list +fi + +apt-get -qq update -y + +# download PKG_URL if PKG_HASH is omitted +mkdir /root/build +cd /root/build + +# clone pbk repo +git clone $PKG_URL ${PKG_NAME}_${PKG_VERSION} +cd ${PKG_NAME}_${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +PG_TOC=$(echo ${PG_VERSION} | sed 's|\.||g') +# Download PostgreSQL source if building for vanilla +if [[ ${PBK_EDITION} == '' ]] ; then + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 +fi + +cd /root/build/${PKG_NAME}_${PKG_VERSION} +cp -av /app/in/specs/deb/pg_probackup/debian ./ +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/changelog + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/g" debian/changelog + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/g" debian/changelog + sed -i "s/@PKG_HASH@/${PKG_HASH}/g" debian/changelog + sed -i "s/@CODENAME@/${CODENAME}/g" debian/changelog + + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/control + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/control + + sed -i "s/@PG_VERSION@/${PG_VERSION}/" debian/pg_probackup.install + mv debian/pg_probackup.install debian/${PKG_NAME}.install + + sed -i "s/@PKG_NAME@/${PKG_NAME}/g" debian/rules + sed -i "s/@PG_TOC@/${PG_TOC}/g" debian/rules + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/rules + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/g" debian/rules + sed -i "s|@PREFIX@|/stump|g" debian/rules +else + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/changelog + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/g" debian/changelog + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/g" debian/changelog + sed -i "s/@PKG_HASH@/${PKG_HASH}/g" debian/changelog + sed -i "s/@CODENAME@/${CODENAME}/g" debian/changelog + + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/control + sed -i "s/pg-probackup-@PG_VERSION@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/control + sed -i "s/@PG_VERSION@/${PG_VERSION}/g" debian/control + sed -i "s/PostgreSQL/PostgresPro ${PBK_EDITION_FULL}/g" debian/control + + sed -i "s/pg_probackup-@PG_VERSION@/pg_probackup-${PBK_EDITION}-${PG_VERSION}/" debian/pg_probackup.install + mv debian/pg_probackup.install debian/pg-probackup-${PBK_EDITION}-${PG_VERSION}.install + + sed -i "s/@PKG_NAME@/pg-probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/rules + sed -i "s/@PG_TOC@/${PG_TOC}/g" debian/rules + sed -i "s/pg_probackup-@PG_VERSION@/pg_probackup-${PBK_EDITION}-${PG_VERSION}/g" debian/rules + sed -i "s/postgresql-@PG_FULL_VERSION@/postgrespro-${PBK_EDITION}-${PG_FULL_VERSION}/g" debian/rules + + if [ ${PG_VERSION} == '9.6' ]; then + sed -i "s|@PREFIX@|/stump|g" debian/rules + else + sed -i "s|@PREFIX@|/opt/pgpro/${PBK_EDITION}-${PG_VERSION}|g" debian/rules + fi +fi + +# Build dependencies +mk-build-deps --install --remove --tool 'apt-get --no-install-recommends --yes' debian/control +rm -rf ./*.deb + +# Pack source to orig.tar.gz +mkdir -p /root/build/dsc +if [[ ${PBK_EDITION} == '' ]] ; then + mv /root/build/postgresql-${PG_FULL_VERSION}.tar.bz2 \ + /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 + + cd /root/build/${PKG_NAME}_${PKG_VERSION} + tar -xf /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 + cd /root/build + + tar -czf ${PKG_NAME}_${PKG_VERSION}.orig.tar.gz \ + ${PKG_NAME}_${PKG_VERSION} + + mv /root/build/${PKG_NAME}_${PKG_VERSION}.orig.tar.gz /root/build/dsc + + cd /root/build/${PKG_NAME}_${PKG_VERSION} + tar -xf /root/build/dsc/${PKG_NAME}_${PKG_VERSION}.orig-postgresql${PG_TOC}.tar.bz2 +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/build/dsc/ + cd /root/build/dsc/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + + mv /root/build/dsc/pgpro /root/build/${PKG_NAME}_${PKG_VERSION}/postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + +# BUILD: SOURCE PKG +if [[ ${PBK_EDITION} == '' ]] ; then + cd /root/build/dsc + dpkg-source -b /root/build/${PKG_NAME}_${PKG_VERSION} +fi + +# BUILD: DEB PKG +cd /root/build/${PKG_NAME}_${PKG_VERSION} +dpkg-buildpackage -b #&> /app/out/build.log + +# COPY ARTEFACTS +rm -rf /app/out/* +cd /root/build +cp -v *.deb /app/out +cp -v *.changes /app/out + +if [[ ${PBK_EDITION} == '' ]] ; then + cp -arv dsc /app/out +fi diff --git a/packaging/pkg/scripts/rpm.sh b/packaging/pkg/scripts/rpm.sh new file mode 100755 index 000000000..2fec4a700 --- /dev/null +++ b/packaging/pkg/scripts/rpm.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + + +#yum upgrade -y || echo "some packages in docker fail to install" +#if [ -f /etc/rosa-release ]; then +# # Avoids old yum bugs on rosa-6 +# yum upgrade -y || echo "some packages in docker fail to install" +#fi + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +if [ ${DISTRIB} = 'centos' ] ; then + sed -i 's|^baseurl=http://|baseurl=https://|g' /etc/yum.repos.d/*.repo + if [ ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi + yum update -y + if [ ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi +fi + +# PACKAGES NEEDED +yum install -y git wget bzip2 rpm-build +if [ ${DISTRIB} = 'oraclelinux' -a ${DISTRIB_VERSION} = '8' -a -n ${PBK_EDITION} ] ; then + yum install -y bison flex +fi + +mkdir /root/build +cd /root/build +rpm --rebuilddb && yum clean all + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} + +# move it to source +cd /root/build +if [[ ${PBK_EDITION} == '' ]] ; then + tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} + mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES + rm -rf pg_probackup-${PKG_VERSION} +else + mv pg_probackup-${PKG_VERSION} /root/rpmbuild/SOURCES +fi + +if [[ ${PBK_EDITION} == '' ]] ; then + + # Download PostgreSQL source + wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O /root/rpmbuild/SOURCES/postgresql-${PG_VERSION}.tar.bz2 + + cd /root/rpmbuild/SOURCES/ + sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup.repo + if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup.repo + elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup.repo + elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup.repo + fi +else + tar -xf /app/in/tarballs/pgpro.tar.bz2 -C /root/rpmbuild/SOURCES/ + cd /root/rpmbuild/SOURCES/pgpro + + PGPRO_TOC=$(echo ${PG_FULL_VERSION} | sed 's|\.|_|g') + if [[ ${PBK_EDITION} == 'std' ]] ; then + git checkout "PGPRO${PGPRO_TOC}_1" + else + git checkout "PGPROEE${PGPRO_TOC}_1" + fi + rm -rf .git + + cd /root/rpmbuild/SOURCES/ + sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup-forks.repo + if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup-forks.repo + elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup-forks.repo + elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup-forks.repo + fi + + mv pgpro postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + chown -R root:root postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} + +# tar -cjf postgrespro-${PBK_EDITION}-${PG_FULL_VERSION}.tar.bz2 postgrespro-${PBK_EDITION}-${PG_FULL_VERSION} +fi + +cd /root/rpmbuild/SPECS +if [[ ${PBK_EDITION} == '' ]] ; then + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.spec + + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo.spec +else + sed -i "s/@EDITION@/${PBK_EDITION}/" pg_probackup-pgpro.spec + sed -i "s/@EDITION_FULL@/${PBK_EDITION_FULL}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-pgpro.spec + sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup-pgpro.spec + sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup-pgpro.spec + sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup-pgpro.spec + + if [ ${PG_VERSION} != '9.6' ]; then + sed -i "s|@PREFIX@|/opt/pgpro/${EDITION}-${PG_VERSION}|g" pg_probackup-pgpro.spec + fi + + sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo-forks.spec + sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo-forks.spec +fi + +if [[ ${PBK_EDITION} == '' ]] ; then + + # install dependencies + yum-builddep -y pg_probackup.spec + + # build pg_probackup + rpmbuild -bs pg_probackup.spec + rpmbuild -ba pg_probackup.spec + + # build repo files + rpmbuild -bs pg_probackup-repo.spec + rpmbuild -ba pg_probackup-repo.spec + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out +else + # install dependencies + yum-builddep -y pg_probackup-pgpro.spec + # build pg_probackup + rpmbuild -ba pg_probackup-pgpro.spec + + # build repo files + rpmbuild -ba pg_probackup-repo-forks.spec + + # write artefacts to out directory + rm -rf /app/out/* + cp -arv /root/rpmbuild/RPMS /app/out +fi diff --git a/packaging/pkg/scripts/suse.sh b/packaging/pkg/scripts/suse.sh new file mode 100755 index 000000000..76b444b5b --- /dev/null +++ b/packaging/pkg/scripts/suse.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + + +#yum upgrade -y || echo "some packages in docker fail to install" +#if [ -f /etc/rosa-release ]; then +# # Avoids old yum bugs on rosa-6 +# yum upgrade -y || echo "some packages in docker fail to install" +#fi + +set -xe +set -o pipefail + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 +zypper clean + +# PACKAGES NEEDED +zypper install -y git wget bzip2 rpm-build + +mkdir /root/build +cd /root/build + +# Copy rpmbuild +cp -rv /app/in/specs/rpm/rpmbuild /root/ + +# download pbk +git clone $PKG_URL pg_probackup-${PKG_VERSION} +cd pg_probackup-${PKG_VERSION} +git checkout ${PKG_HASH} +cd .. + +# tarball it +tar -cjf pg_probackup-${PKG_VERSION}.tar.bz2 pg_probackup-${PKG_VERSION} +mv pg_probackup-${PKG_VERSION}.tar.bz2 /root/rpmbuild/SOURCES +rm -rf pg_probackup-${PKG_VERSION} + +# Download PostgreSQL source +wget -q http://ftp.postgresql.org/pub/source/v${PG_FULL_VERSION}/postgresql-${PG_FULL_VERSION}.tar.bz2 -O /root/rpmbuild/SOURCES/postgresql-${PG_VERSION}.tar.bz2 + +rm -rf /usr/src/packages +ln -s /root/rpmbuild /usr/src/packages + +cd /root/rpmbuild/SOURCES +sed -i "s/@PG_VERSION@/${PKG_VERSION}/" pg_probackup.repo + + +# change to build dir +cd /root/rpmbuild/SOURCES +sed -i "s/@DISTRIB@/${DISTRIB}/" pg_probackup.repo +if [ $DISTRIB == 'centos' ] + then sed -i "s/@SHORT_CODENAME@/Centos/" pg_probackup.repo +elif [ $DISTRIB == 'rhel' ] + then sed -i "s/@SHORT_CODENAME@/RedHat/" pg_probackup.repo +elif [ $DISTRIB == 'oraclelinux' ] + then sed -i "s/@SHORT_CODENAME@/Oracle/" pg_probackup.repo +elif [ $DISTRIB == 'suse' ] + then sed -i "s/@SHORT_CODENAME@/SUSE/" pg_probackup.repo +fi + +cd /root/rpmbuild/SPECS +sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup.spec +sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup.spec +sed -i "s/@PKG_HASH@/${PKG_HASH}/" pg_probackup.spec +sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup.spec +sed -i "s/@PG_FULL_VERSION@/${PG_FULL_VERSION}/" pg_probackup.spec + +sed -i "s/@PG_VERSION@/${PG_VERSION}/" pg_probackup-repo.spec +sed -i "s/@PKG_VERSION@/${PKG_VERSION}/" pg_probackup-repo.spec +sed -i "s/@PKG_RELEASE@/${PKG_RELEASE}/" pg_probackup-repo.spec + +# install dependencies +zypper -n install \ + $(rpmspec --parse pg_probackup.spec | grep BuildRequires | cut -d':' -f2 | xargs) + +# build pg_probackup +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log + +# build repo files, TODO: move to separate repo +rpmbuild -ba pg_probackup-repo.spec + +# write artefacts to out directory +rm -rf /app/out/* + +cp -arv /root/rpmbuild/{RPMS,SRPMS} /app/out diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/changelog b/packaging/pkg/specs/deb/pg_probackup/debian/changelog new file mode 100644 index 000000000..5b9160220 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/changelog @@ -0,0 +1,11 @@ +@PKG_NAME@ (@PKG_VERSION@-@PKG_RELEASE@.@PKG_HASH@.@CODENAME@) @CODENAME@; urgency=medium + + * @PKG_VERSION@ + + -- Grigory Smolkin Wed, 9 Feb 2018 10:22:08 +0300 + +@PKG_NAME@ (2.0.14-1.@CODENAME@) @CODENAME@; urgency=medium + + * Initial package + + -- Grigory Smolkin Fri, 29 Jan 2018 10:22:08 +0300 diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/compat b/packaging/pkg/specs/deb/pg_probackup/debian/compat new file mode 100644 index 000000000..ec635144f --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/compat @@ -0,0 +1 @@ +9 diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/control b/packaging/pkg/specs/deb/pg_probackup/debian/control new file mode 100644 index 000000000..8f1d42007 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/control @@ -0,0 +1,29 @@ +Source: @PKG_NAME@ +Section: database +Priority: optional +Maintainer: PostgresPro DBA +Uploaders: Grigory Smolkin +Build-Depends: + debhelper (>= 9), + bison, + dpkg-dev, + flex, + gettext, + zlib1g-dev | libz-dev, + libpq5 +Standards-Version: 3.9.6 +Homepage: https://github.com/postgrespro/pg_probackup + +Package: @PKG_NAME@ +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: Backup tool for PostgreSQL. + . + This package provides pg_probackup binary for PostgreSQL @PG_VERSION@. + +Package: @PKG_NAME@-dbg +Depends: @PKG_NAME@ +Architecture: any +Description: Backup tool for PostgreSQL. + . + This package provides detached debugging symbols for pg_probackup diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install b/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install new file mode 100644 index 000000000..ed904ca40 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/pg_probackup.install @@ -0,0 +1 @@ +pg_probackup-@PG_VERSION@ /usr/bin/ \ No newline at end of file diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/rules b/packaging/pkg/specs/deb/pg_probackup/debian/rules new file mode 100644 index 000000000..309a9a1d4 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/rules @@ -0,0 +1,29 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ + +override_dh_auto_clean: + # skip + +override_dh_auto_build: + cd postgresql-@PG_FULL_VERSION@ && ./configure --enable-debug --without-readline --prefix=@PREFIX@ &&\ + make MAKELEVEL=0 install DESTDIR=$(CURDIR)/debian/tmp && cd .. &&\ + make USE_PGXS=1 top_srcdir=$(CURDIR)/postgresql-@PG_FULL_VERSION@ PG_CONFIG=$(CURDIR)/debian/tmp/@PREFIX@/bin/pg_config &&\ + mv pg_probackup pg_probackup-@PG_VERSION@ + +override_dh_auto_test: + # skip + +override_dh_auto_install: + # skip + +override_dh_strip: + dh_strip --dbg-package=@PKG_NAME@-dbg + +override_dh_auto_clean: + # skip + #make clean top_srcdir=$(CURDIR)/pg@PG_TOC@-source PG_CONFIG=$(CURDIR)/debian/tmp/stump/bin/pg_config diff --git a/packaging/pkg/specs/deb/pg_probackup/debian/source/format b/packaging/pkg/specs/deb/pg_probackup/debian/source/format new file mode 100644 index 000000000..163aaf8d8 --- /dev/null +++ b/packaging/pkg/specs/deb/pg_probackup/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP new file mode 100644 index 000000000..c11d9c015 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFpy9DABEADd44hR3o4i4DrUephrr7iHPHcRH0Zego3A36NdOf0ymP94H8Bi +U8C6YyKFbltShh18IC3QZJK04hLRQEs6sPKC2XHwlz+Tndi49Z45pfV54xEVKmBS +IZ5AM9y1FxwQAOzu6pZGu32DWDXZzhI7nLuY8rqAMMuzKeRcGm3sQ6ZcAwYOLT+e +ZAxkUL05MBGDaLc91HtKiurRlHuMySiVdkNu9ebTGV4zZv+ocBK8iC5rJjTJCv78 +eLkrRgjp7/MuLQ7mmiwfZx5lUIO9S87HDeH940mcYWRGUsdCbj0791wHY0PXlqhH +6lCLGur9/5yM88pGU79uahCblxsYdue6fdgIZR0hQBUxnLphI2SCshP89VDEwaP2 +dlC/qESJ3xyULkdJz67wlhOrIPC9T1d2pa5MUurOK0yTFH7j4JLWjBgU59h31ZEF +NMHde+Fwv+lL/yRht2Xz7HG5Rt8ogn4/rPBloXr1v83iN34aZnnqanyhSbE9xUhP +RNK3fBxXmX9IjFsBhRelPcv5NWNnxnnMkEfhoZvrAy+ykUGLP+J+Rj+d5v/8nAUc +taxqAXlUz1VabR0BVISBsRY+ket4O2dJ1WbZ8KXG6q/F9UMpS0v9aRdb1JyzrWCw +wT/l3q9x89i27SgDZgAfEFhvbMN6hUmFyVoMBgk8kqvi4b3lZZGCeuLX5wARAQAB +tCxQb3N0Z3JlU1FMIFByb2Zlc3Npb25hbCA8ZGJhQHBvc3RncmVzcHJvLnJ1PokC +OQQTAQIAIwUCWnL0MAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEKeJ +efZjbXF+zDUP/RfYxlq3erzP/cG6/LghZlJy6hGuUgyDFj2zUVAbpoFhqCAmaNLc ++bBYMCyNRhS8/oXushCSxUV8D7LRIRIRdtbNAnd4MNl6U4ORF6JcdPPNLROzwMik +3TmIVACMdjb9IRF5+8jVrIgDPI/FVtf5qp0Ot6OBtpD5oWQ7ubZ31RPR3pacdujK +jlbzL5Y6HsonhMbSJU/d0d9DylMvX4Gcxdw7M2Pfe3E6mjPJmcHiKuCKln2eLOsg +53HA/RWKy+uYDo+vdefUsQCIdnC5VghnXG8FTuvVqeqiSeU2XdyuzjndqxKZNrMw +YK1POK7R55R1aKJaOKEwnfd5oN02p77U+R/vb/mDcfZWbXI8JrHwPKVOQiEl0S+G +ePPW57EmX9yFuWAzcOPp9yCt/+roVry1ICifrFaLOhtT+/vle0j3+rbn31BMPsjf +QbREVetHfWB0N78k/hKC8SivDdrXsdqovcGgSAjFzPEdznvx9vKopwz2CQ6DK25Q +3M4j79Akcaa08k5Wphyx48PbhzSeE/d4xVzey7ge0BwYMdNGXKeyBjT6h9e+iySE +UTZ3/3c7O1D8p2EfPUMT/aI5fWlLBXlT5fDp2yX0HMTt/NUIXAiTHb5BDnZ+4ld3 +KXjHw4WzaOfHBfGDjJDtHPgdTEJTsQbH8//D+wwU3ueNS1ho4DpLqc+YuQINBFpy +9DABEADJMkgQ2m4g4LX7FNnmQbRgDcuhL8Y0VRGST+5x6xvb2em1boQHUaTt7/3z +DnaIRrZqrFP09O6xblSjEu9FZE+JuQGNyC4TH9fjvKnkRlqTF6X87nRVGByRmrdL +lPp9XPJY2Mc7c0PisncI/j7d9PmUHOSmaWeLG/WqMbzZA+s1IWjC0tqIN2k5ivTN +PfRm+9ebEHMUN+D7yZQMBlCmFexwy6h5pAioyj4tAOHqxfNDE33qezaeBn/E1BpW +NyegKwNtPUL0t2kXTO5tspKKCcny4HJ7K60gak0fWp42qVygwSPR54ztFM+6XjCh +0MmZ/mAdzLd6OJiP8RfMCfXbXpK4793+Cw0AK3Mu+mnJ26kz1KEZ9DKiAEhBhK3r +Z3/isUc8LcVYLHIduH9b/K50FjgR0T1Lm4r6Hpf6nTROlfiFSMXJU0HepAzMPHRq +EWqTJ49UgI7Llf+aBP7fGLqRPvWJpAJaQkMiUxfP5JYYCb+45d7I54iXQCD6ToK1 +bDnh+zZIrwyUIxPfFQh1xPYyFWRELJpeOFzm+espqiVFPXpBoimVlytwNrGdbxbY +SO0eEVlE41AjD8cgk+ibAvt/moT2+Mps/t083LR+J92kj+iX/D4NHVy4CjJTrhwO +rI3FrxtdU+NFXULyj0KslOKuyG5WuHLQvfL5P3JGuTkP4iJOTQARAQABiQIfBBgB +AgAJBQJacvQwAhsMAAoJEKeJefZjbXF+8JgQAJqlO1ftIsJvZ/+4ZVVOTPx5ZmYs +ABp4/2gaiLdhajN8ynbZqtCyjtQwSCLJFf2CcDL8XUooJzdQECkqdiI7ouYSFBzO +ui3jjCuFz5oHv88OtX2cIRxHqlZQmXEHvk0gH61xDV5CWBJmjxdRcsC7n1I8DSVg +Qmuq06S+xIX6rHf2CRxYKahBip71u7OIH4BRV44y26xf1a8an+8BkqF9+mYt7zqO +vyMCJ1UftXcuE5SxY54jnNAavF7Kq/2Yp7v3aYqFREngxtbWudyo7QW5EuToSvY2 +qY6tpInahWjuXxeARsFzp4fB0Eo/yH+iqG30zkQCuxLyxzbMMcNQP4if3yV6uO14 +LqapZLrMp6IMTfHDKmbbtDQ2RpRRut3K4khXRQ1LjGKziOU4ZCEazrXEijm2AlKw +7JS3POGvM+VAiaGNBqfdHpTwXXT7zkxJjfJC3Et/6fHy1xuCtUtMs41PjHS/HWi8 +w70T8XpKub+ewVElxq2D83rx07w3HuBtVUyqG0XgcACwqQA1vMLJaR3VoX1024ho +sf2PtZqQ7SCgt0hkZAT72j05nz4bIxUIcDkAGtd9FDPQ4Ixi6fRfTJpZ7lIEV5as +Zs9C0hrxmWgJwSGgQa2Waylvw47fMwfMn+gUNRqwanyOjVYfpSJafLc6Ol43bQN/ +jCKs4enncezhjcAh +=TVZj +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS new file mode 100644 index 000000000..c11d9c015 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/GPG-KEY-PG_PROBACKUP-FORKS @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFpy9DABEADd44hR3o4i4DrUephrr7iHPHcRH0Zego3A36NdOf0ymP94H8Bi +U8C6YyKFbltShh18IC3QZJK04hLRQEs6sPKC2XHwlz+Tndi49Z45pfV54xEVKmBS +IZ5AM9y1FxwQAOzu6pZGu32DWDXZzhI7nLuY8rqAMMuzKeRcGm3sQ6ZcAwYOLT+e +ZAxkUL05MBGDaLc91HtKiurRlHuMySiVdkNu9ebTGV4zZv+ocBK8iC5rJjTJCv78 +eLkrRgjp7/MuLQ7mmiwfZx5lUIO9S87HDeH940mcYWRGUsdCbj0791wHY0PXlqhH +6lCLGur9/5yM88pGU79uahCblxsYdue6fdgIZR0hQBUxnLphI2SCshP89VDEwaP2 +dlC/qESJ3xyULkdJz67wlhOrIPC9T1d2pa5MUurOK0yTFH7j4JLWjBgU59h31ZEF +NMHde+Fwv+lL/yRht2Xz7HG5Rt8ogn4/rPBloXr1v83iN34aZnnqanyhSbE9xUhP +RNK3fBxXmX9IjFsBhRelPcv5NWNnxnnMkEfhoZvrAy+ykUGLP+J+Rj+d5v/8nAUc +taxqAXlUz1VabR0BVISBsRY+ket4O2dJ1WbZ8KXG6q/F9UMpS0v9aRdb1JyzrWCw +wT/l3q9x89i27SgDZgAfEFhvbMN6hUmFyVoMBgk8kqvi4b3lZZGCeuLX5wARAQAB +tCxQb3N0Z3JlU1FMIFByb2Zlc3Npb25hbCA8ZGJhQHBvc3RncmVzcHJvLnJ1PokC +OQQTAQIAIwUCWnL0MAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEKeJ +efZjbXF+zDUP/RfYxlq3erzP/cG6/LghZlJy6hGuUgyDFj2zUVAbpoFhqCAmaNLc ++bBYMCyNRhS8/oXushCSxUV8D7LRIRIRdtbNAnd4MNl6U4ORF6JcdPPNLROzwMik +3TmIVACMdjb9IRF5+8jVrIgDPI/FVtf5qp0Ot6OBtpD5oWQ7ubZ31RPR3pacdujK +jlbzL5Y6HsonhMbSJU/d0d9DylMvX4Gcxdw7M2Pfe3E6mjPJmcHiKuCKln2eLOsg +53HA/RWKy+uYDo+vdefUsQCIdnC5VghnXG8FTuvVqeqiSeU2XdyuzjndqxKZNrMw +YK1POK7R55R1aKJaOKEwnfd5oN02p77U+R/vb/mDcfZWbXI8JrHwPKVOQiEl0S+G +ePPW57EmX9yFuWAzcOPp9yCt/+roVry1ICifrFaLOhtT+/vle0j3+rbn31BMPsjf +QbREVetHfWB0N78k/hKC8SivDdrXsdqovcGgSAjFzPEdznvx9vKopwz2CQ6DK25Q +3M4j79Akcaa08k5Wphyx48PbhzSeE/d4xVzey7ge0BwYMdNGXKeyBjT6h9e+iySE +UTZ3/3c7O1D8p2EfPUMT/aI5fWlLBXlT5fDp2yX0HMTt/NUIXAiTHb5BDnZ+4ld3 +KXjHw4WzaOfHBfGDjJDtHPgdTEJTsQbH8//D+wwU3ueNS1ho4DpLqc+YuQINBFpy +9DABEADJMkgQ2m4g4LX7FNnmQbRgDcuhL8Y0VRGST+5x6xvb2em1boQHUaTt7/3z +DnaIRrZqrFP09O6xblSjEu9FZE+JuQGNyC4TH9fjvKnkRlqTF6X87nRVGByRmrdL +lPp9XPJY2Mc7c0PisncI/j7d9PmUHOSmaWeLG/WqMbzZA+s1IWjC0tqIN2k5ivTN +PfRm+9ebEHMUN+D7yZQMBlCmFexwy6h5pAioyj4tAOHqxfNDE33qezaeBn/E1BpW +NyegKwNtPUL0t2kXTO5tspKKCcny4HJ7K60gak0fWp42qVygwSPR54ztFM+6XjCh +0MmZ/mAdzLd6OJiP8RfMCfXbXpK4793+Cw0AK3Mu+mnJ26kz1KEZ9DKiAEhBhK3r +Z3/isUc8LcVYLHIduH9b/K50FjgR0T1Lm4r6Hpf6nTROlfiFSMXJU0HepAzMPHRq +EWqTJ49UgI7Llf+aBP7fGLqRPvWJpAJaQkMiUxfP5JYYCb+45d7I54iXQCD6ToK1 +bDnh+zZIrwyUIxPfFQh1xPYyFWRELJpeOFzm+espqiVFPXpBoimVlytwNrGdbxbY +SO0eEVlE41AjD8cgk+ibAvt/moT2+Mps/t083LR+J92kj+iX/D4NHVy4CjJTrhwO +rI3FrxtdU+NFXULyj0KslOKuyG5WuHLQvfL5P3JGuTkP4iJOTQARAQABiQIfBBgB +AgAJBQJacvQwAhsMAAoJEKeJefZjbXF+8JgQAJqlO1ftIsJvZ/+4ZVVOTPx5ZmYs +ABp4/2gaiLdhajN8ynbZqtCyjtQwSCLJFf2CcDL8XUooJzdQECkqdiI7ouYSFBzO +ui3jjCuFz5oHv88OtX2cIRxHqlZQmXEHvk0gH61xDV5CWBJmjxdRcsC7n1I8DSVg +Qmuq06S+xIX6rHf2CRxYKahBip71u7OIH4BRV44y26xf1a8an+8BkqF9+mYt7zqO +vyMCJ1UftXcuE5SxY54jnNAavF7Kq/2Yp7v3aYqFREngxtbWudyo7QW5EuToSvY2 +qY6tpInahWjuXxeARsFzp4fB0Eo/yH+iqG30zkQCuxLyxzbMMcNQP4if3yV6uO14 +LqapZLrMp6IMTfHDKmbbtDQ2RpRRut3K4khXRQ1LjGKziOU4ZCEazrXEijm2AlKw +7JS3POGvM+VAiaGNBqfdHpTwXXT7zkxJjfJC3Et/6fHy1xuCtUtMs41PjHS/HWi8 +w70T8XpKub+ewVElxq2D83rx07w3HuBtVUyqG0XgcACwqQA1vMLJaR3VoX1024ho +sf2PtZqQ7SCgt0hkZAT72j05nz4bIxUIcDkAGtd9FDPQ4Ixi6fRfTJpZ7lIEV5as +Zs9C0hrxmWgJwSGgQa2Waylvw47fMwfMn+gUNRqwanyOjVYfpSJafLc6Ol43bQN/ +jCKs4enncezhjcAh +=TVZj +-----END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo new file mode 100644 index 000000000..d26b058cd --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup-forks.repo @@ -0,0 +1,6 @@ +[pg_probackup-forks] +name=PG_PROBACKUP @SHORT_CODENAME@ packages for PostgresPro Standard and Enterprise - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP diff --git a/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo new file mode 100644 index 000000000..33dc31a24 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SOURCES/pg_probackup.repo @@ -0,0 +1,13 @@ +[pg_probackup] +name=PG_PROBACKUP Packages for @SHORT_CODENAME@ Linux - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup/rpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +[pg_probackup-sources] +name=PG_PROBACKUP Source Packages for @SHORT_CODENAME@ Linux - $basearch +baseurl=https://repo.postgrespro.ru/pg_probackup/srpm/latest/@DISTRIB@-$releasever-$basearch +enabled=1 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec new file mode 100644 index 000000000..8955b3fa7 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-pgpro.spec @@ -0,0 +1,71 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%global edition @EDITION@ +%global edition_full @EDITION_FULL@ +%global prefix @PREFIX@ + +Name: pg_probackup-%{edition}-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgresPro %{edition_full} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +#Source0: postgrespro-%{edition}-%{pgsql_full}.tar.bz2 +#Source1: pg_probackup-%{version}.tar.bz2 +Source0: postgrespro-%{edition}-%{pgsql_full} +Source1: pg_probackup-%{version} +BuildRequires: gcc make perl glibc-devel +BuildRequires: openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgresPro %{edition_full}. + +%prep +#%setup -q -b1 -n pg_probackup-%{version}.tar.bz2 +mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} %{_topdir}/BUILD +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup + +mkdir %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +mkdir %{_topdir}/SOURCES/pg_probackup-%{version} + +%build +#cd %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +#mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} ./ +#cd postgrespro-%{edition}-%{pgsql_full} +#mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} + +%if "%{pgsql_major}" == "9.6" +./configure --enable-debug --without-readline +%else +./configure --enable-debug --without-readline --prefix=%{prefix} +%endif +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Wed Feb 9 2018 Grigory Smolkin - %{version}-%{release}.%{hash} +- @PKG_VERSION@ + +* Fri Jan 29 2018 Grigory Smolkin - 2.0.14-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec new file mode 100644 index 000000000..47adb250f --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo-forks.spec @@ -0,0 +1,49 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ + +Summary: pg_probackup repo RPM +Name: pg_probackup-repo-forks +Version: %{version} +Release: %{release} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ + +Source0: http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP +Source1: pg_probackup-forks.repo + +BuildArch: noarch + +%description +This package contains yum configuration for @SHORT_CODENAME@, and also the GPG key +for pg_probackup RPMs for PostgresPro Standard and Enterprise. + +%prep +%setup -q -c -T +install -pm 644 %{SOURCE0} . +install -pm 644 %{SOURCE1} . + +%build + +%install +rm -rf $RPM_BUILD_ROOT + +#GPG Key +install -Dpm 644 %{SOURCE0} \ + $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +# yum +install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d +install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%config(noreplace) /etc/yum.repos.d/* +/etc/pki/rpm-gpg/* + +%changelog +* Fri Oct 26 2019 Grigory Smolkin +- Initial package diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec new file mode 100644 index 000000000..da54bc7b1 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup-repo.spec @@ -0,0 +1,58 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ + +Summary: PG_PROBACKUP RPMs +Name: pg_probackup-repo +Version: %{version} +Release: %{release} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ + +Source0: http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP +Source1: pg_probackup.repo + +BuildArch: noarch + +%description +This package contains yum configuration for Centos, and also the GPG key for PG_PROBACKUP RPMs. + +%prep +%setup -q -c -T +install -pm 644 %{SOURCE0} . +install -pm 644 %{SOURCE1} . + +%build + +%install +rm -rf $RPM_BUILD_ROOT + +#GPG Key +install -Dpm 644 %{SOURCE0} \ + $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/GPG-KEY-PG_PROBACKUP + +# yum /etc/zypp/repos.d/repo-update.repo + +%if 0%{?suse_version} + install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d + install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/zypp/repos.d +%else + install -dm 755 $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d + install -pm 644 %{SOURCE1} $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d +%endif + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%if 0%{?suse_version} + %config(noreplace) /etc/zypp/repos.d/* +%else + %config(noreplace) /etc/yum.repos.d/* +%endif +/etc/pki/rpm-gpg/* + +%changelog +* Mon Jun 29 2020 Grigory Smolkin +- release update diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec new file mode 100644 index 000000000..cbb57e42a --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.forks.spec @@ -0,0 +1,67 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%global edition @EDITION@ +%global edition_full @EDITION_FULL@ +%global prefix @PREFIX@ + +#%set_verify_elf_method unresolved=relaxed, rpath=relaxed +%set_verify_elf_method rpath=relaxed,unresolved=relaxed + +Name: pg_probackup-%{edition}-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgresPro %{edition_full} +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +#Source0: postgrespro-%{edition}-%{pgsql_full}.tar.bz2 +#Source1: pg_probackup-%{edition}-%{version}.tar.bz2 +Source0: postgrespro-%{edition}-%{pgsql_full} +Source1: pg_probackup-%{version} +BuildRequires: gcc make perl glibc-devel bison flex +BuildRequires: readline-devel openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgresPro %{edition_full}. + +%prep +#%setup -q -b1 -n postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} %{_topdir}/BUILD +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +mv %{_topdir}/SOURCES/pg_probackup-%{version} contrib/pg_probackup + +mkdir %{_topdir}/SOURCES/postgrespro-%{edition}-%{pgsql_full} +mkdir %{_topdir}/SOURCES/pg_probackup-%{edition}-%{version} +mkdir %{_topdir}/SOURCES/pg_probackup-%{version} + +%build +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%if "%{pgsql_major}" == "9.6" +./configure --enable-debug +%else +./configure --enable-debug --disable-online-upgrade --prefix=%{prefix} +%endif +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +cd %{_topdir}/BUILD/postgrespro-%{edition}-%{pgsql_full} +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Mon Nov 17 2019 Grigory Smolkin - 2.2.6-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec new file mode 100644 index 000000000..3105ffa67 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.alt.spec @@ -0,0 +1,48 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ +%set_verify_elf_method rpath=relaxed + +Name: pg_probackup-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgreSQL +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +Source0: http://ftp.postgresql.org/pub/source/v%{pgsql_full}/postgresql-%{pgsql_major}.tar.bz2 +Source1: pg_probackup-%{version}.tar.bz2 +BuildRequires: gcc make perl glibc-devel bison flex +BuildRequires: readline-devel openssl-devel gettext zlib-devel + + +%description +Backup tool for PostgreSQL. + +%prep +%setup -q -b1 -n postgresql-%{pgsql_full} + +%build +mv %{_builddir}/pg_probackup-%{version} contrib/pg_probackup +./configure --enable-debug --without-readline +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Mon Nov 17 2019 Grigory Smolkin - 2.2.6-1 +- Initial release. diff --git a/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec new file mode 100644 index 000000000..e5fb5ad48 --- /dev/null +++ b/packaging/pkg/specs/rpm/rpmbuild/SPECS/pg_probackup.spec @@ -0,0 +1,48 @@ +%global version @PKG_VERSION@ +%global release @PKG_RELEASE@ +%global hash @PKG_HASH@ +%global pgsql_major @PG_VERSION@ +%global pgsql_full @PG_FULL_VERSION@ + +Name: pg_probackup-%{pgsql_major} +Version: %{version} +Release: %{release}.%{hash} +Summary: Backup utility for PostgreSQL +Group: Applications/Databases +License: BSD +Url: http://postgrespro.ru/ +Source0: http://ftp.postgresql.org/pub/source/v%{pgsql_full}/postgresql-%{pgsql_major}.tar.bz2 +Source1: pg_probackup-%{version}.tar.bz2 +BuildRequires: gcc make perl glibc-devel openssl-devel gettext zlib-devel + +%description +Backup tool for PostgreSQL. + +%prep +%setup -q -b1 -n postgresql-%{pgsql_full} + +%build +mv %{_builddir}/pg_probackup-%{version} contrib/pg_probackup +./configure --enable-debug --without-readline +make -C 'src/common' +make -C 'src/port' +make -C 'src/interfaces' +cd contrib/pg_probackup && make + +%install +%{__mkdir} -p %{buildroot}%{_bindir} +%{__install} -p -m 755 contrib/pg_probackup/pg_probackup %{buildroot}%{_bindir}/%{name} + +%files +%{_bindir}/%{name} + +%clean +rm -rf $RPM_BUILD_ROOT + + +%changelog +* Wed Feb 9 2018 Grigory Smolkin - %{version}-%{release}.%{hash} +- @PKG_VERSION@ + +* Fri Jan 29 2018 Grigory Smolkin - 2.0.14-1 +- Initial release. diff --git a/packaging/pkg/tarballs/.gitkeep b/packaging/pkg/tarballs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/packaging/repo/reprepro-conf/changelog.script b/packaging/repo/reprepro-conf/changelog.script new file mode 100755 index 000000000..4ff1f1787 --- /dev/null +++ b/packaging/repo/reprepro-conf/changelog.script @@ -0,0 +1,246 @@ +#!/bin/sh +# This is an example script that can be hooked into reprepro +# to either generate a hierachy like packages.debian.org/changelogs/ +# or to generate changelog files in the "third party sites" +# location apt-get changelogs looks if it is not found in +# Apt::Changelogs::Server. +# +# All you have to do is to: +# - copy it into you conf/ directory, +# - if you want "third party site" style changelogs, edit the +# CHANGELOGDIR variable below, +# and +# - add the following to any distribution in conf/distributions +# you want to have changelogs and copyright files extracted: +#Log: +# --type=dsc changelogs.example +# (note the space at the beginning of the second line). +# This will cause this script to extract changelogs for all +# newly added source packages. (To generate them for already +# existing packages, call "reprepro rerunnotifiers"). + +# DEPENDENCIES: dpkg >= 1.13.9 + +if test "x${REPREPRO_OUT_DIR:+set}" = xset ; then + # Note: due to cd, REPREPRO_*_DIR will no longer + # be usable. And only things relative to outdir will work... + cd "${REPREPRO_OUT_DIR}" || exit 1 +else + # this will also trigger if reprepro < 3.5.1 is used, + # in that case replace this with a manual cd to the + # correct directory... + cat "changelog.example needs to be run by reprepro!" >&2 + exit 1 +fi + +# CHANGELOGDIR set means generate full hierachy +# (clients need to set Apt::Changelogs::Server to use that) +#CHANGELOGDIR=changelogs + +# CHANGELOGDIR empty means generate changelog (and only changelog) files +# in the new "third party site" place apt-get changelog is using as fallback: +#CHANGELOGDIR= + +# Set to avoid using some predefined TMPDIR or even /tmp as +# tempdir: + +# TMPDIR=/var/cache/whateveryoucreated + +if test -z "$CHANGELOGDIR" ; then +addsource() { + DSCFILE="$1" + CANONDSCFILE="$(readlink --canonicalize "$DSCFILE")" + CHANGELOGFILE="${DSCFILE%.dsc}.changelog" + BASEDIR="$(dirname "$CHANGELOGFILE")" + if ! [ -f "$CHANGELOGFILE" ] ; then + EXTRACTDIR="$(mktemp -d)" + (cd -- "$EXTRACTDIR" && dpkg-source --no-copy -x "$CANONDSCFILE" > /dev/null) + install --mode=644 -- "$EXTRACTDIR"/*/debian/changelog "$CHANGELOGFILE" + chmod -R u+rwX -- "$EXTRACTDIR" + rm -r -- "$EXTRACTDIR" + fi + if [ -L "$BASEDIR"/current."$CODENAME" ] ; then + # should not be there, just to be sure + rm -f -- "$BASEDIR"/current."$CODENAME" + fi + # mark this as needed by this distribution + ln -s -- "$(basename "$CHANGELOGFILE")" "$BASEDIR/current.$CODENAME" + JUSTADDED="$CHANGELOGFILE" +} +delsource() { + DSCFILE="$1" + CHANGELOGFILE="${DSCFILE%.dsc}.changelog" + BASEDIR="$(dirname "$CHANGELOGFILE")" + BASENAME="$(basename "$CHANGELOGFILE")" + if [ "x$JUSTADDED" = "x$CHANGELOGFILE" ] ; then + exit 0 + fi +# echo "delete, basedir=$BASEDIR changelog=$CHANGELOGFILE, dscfile=$DSCFILE, " + if [ "x$(readlink "$BASEDIR/current.$CODENAME")" = "x$BASENAME" ] ; then + rm -- "$BASEDIR/current.$CODENAME" + fi + NEEDED=0 + for c in "$BASEDIR"/current.* ; do + if [ "x$(readlink -- "$c")" = "x$BASENAME" ] ; then + NEEDED=1 + fi + done + if [ "$NEEDED" -eq 0 -a -f "$CHANGELOGFILE" ] ; then + rm -r -- "$CHANGELOGFILE" + # to remove the directory if now empty + rmdir --ignore-fail-on-non-empty -- "$BASEDIR" + fi +} + +else # "$CHANGELOGDIR" set: + +addsource() { + DSCFILE="$1" + CANONDSCFILE="$(readlink --canonicalize "$DSCFILE")" + TARGETDIR="${CHANGELOGDIR}/${DSCFILE%.dsc}" + SUBDIR="$(basename $TARGETDIR)" + BASEDIR="$(dirname $TARGETDIR)" + if ! [ -d "$TARGETDIR" ] ; then + echo "extract $CANONDSCFILE information to $TARGETDIR" + mkdir -p -- "$TARGETDIR" + EXTRACTDIR="$(mktemp -d)" + (cd -- "$EXTRACTDIR" && dpkg-source --no-copy -x "$CANONDSCFILE" > /dev/null) + install --mode=644 -- "$EXTRACTDIR"/*/debian/copyright "$TARGETDIR/copyright" + install --mode=644 -- "$EXTRACTDIR"/*/debian/changelog "$TARGETDIR/changelog" + chmod -R u+rwX -- "$EXTRACTDIR" + rm -r -- "$EXTRACTDIR" + fi + if [ -L "$BASEDIR"/current."$CODENAME" ] ; then + # should not be there, just to be sure + rm -f -- "$BASEDIR"/current."$CODENAME" + fi + # mark this as needed by this distribution + ln -s -- "$SUBDIR" "$BASEDIR/current.$CODENAME" + JUSTADDED="$TARGETDIR" +} +delsource() { + DSCFILE="$1" + TARGETDIR="${CHANGELOGDIR}/${DSCFILE%.dsc}" + SUBDIR="$(basename $TARGETDIR)" + BASEDIR="$(dirname $TARGETDIR)" + if [ "x$JUSTADDED" = "x$TARGETDIR" ] ; then + exit 0 + fi +# echo "delete, basedir=$BASEDIR targetdir=$TARGETDIR, dscfile=$DSCFILE, " + if [ "x$(readlink "$BASEDIR/current.$CODENAME")" = "x$SUBDIR" ] ; then + rm -- "$BASEDIR/current.$CODENAME" + fi + NEEDED=0 + for c in "$BASEDIR"/current.* ; do + if [ "x$(readlink -- "$c")" = "x$SUBDIR" ] ; then + NEEDED=1 + fi + done + if [ "$NEEDED" -eq 0 -a -d "$TARGETDIR" ] ; then + rm -r -- "$TARGETDIR" + # to remove the directory if now empty + rmdir --ignore-fail-on-non-empty -- "$BASEDIR" + fi +} +fi # CHANGELOGDIR + +ACTION="$1" +CODENAME="$2" +PACKAGETYPE="$3" +if [ "x$PACKAGETYPE" != "xdsc" ] ; then +# the --type=dsc should cause this to never happen, but better safe than sorry. + exit 1 +fi +COMPONENT="$4" +ARCHITECTURE="$5" +if [ "x$ARCHITECTURE" != "xsource" ] ; then + exit 1 +fi +NAME="$6" +shift 6 +JUSTADDED="" +if [ "x$ACTION" = "xadd" -o "x$ACTION" = "xinfo" ] ; then + VERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + addsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +elif [ "x$ACTION" = "xremove" ] ; then + OLDVERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + delsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +elif [ "x$ACTION" = "xreplace" ] ; then + VERSION="$1" + shift + OLDVERSION="$1" + shift + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 -a "x$1" != "x--" ] ; do + case "$1" in + *.dsc) + addsource "$1" + ;; + esac + shift + done + if [ "x$1" != "x--" ] ; then + exit 2 + fi + shift + while [ "$#" -gt 0 ] ; do + case "$1" in + *.dsc) + delsource "$1" + ;; + --) + exit 2 + ;; + esac + shift + done +fi + +exit 0 +# Copyright 2007,2008,2012 Bernhard R. Link +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA \ No newline at end of file diff --git a/packaging/repo/reprepro-conf/distributions b/packaging/repo/reprepro-conf/distributions new file mode 100644 index 000000000..7dce7e6d0 --- /dev/null +++ b/packaging/repo/reprepro-conf/distributions @@ -0,0 +1,179 @@ +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: squeeze +Architectures: amd64 i386 source +Components: main-squeeze +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: wheezy +Architectures: amd64 i386 source +Components: main-wheezy +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: jessie +Architectures: amd64 i386 source +Components: main-jessie +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: bullseye +Architectures: amd64 i386 source +Components: main-bullseye +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: wily +Architectures: amd64 i386 source +Components: main-wily +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: precise +Architectures: amd64 i386 source +Components: main-precise +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: vivid +Architectures: amd64 i386 source +Components: main-vivid +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: trusty +Architectures: amd64 i386 source +Components: main-trusty +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: lucid +Architectures: amd64 i386 source +Components: main-lucid +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: cosmic +Architectures: amd64 i386 source +Components: main-cosmic +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: xenial +Architectures: amd64 i386 source +Components: main-xenial +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: yakkety +Architectures: amd64 i386 source +Components: main-yakkety +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: zesty +Architectures: amd64 i386 source +Components: main-zesty +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: stretch +Architectures: amd64 i386 source +Components: main-stretch +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: buster +Architectures: amd64 i386 source +Components: main-buster +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: artful +Architectures: amd64 i386 source +Components: main-artful +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: bionic +Architectures: amd64 i386 source +Components: main-bionic +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script + +Origin: repo.postgrespro.ru +Label: PostgreSQL backup utility pg_probackup +Codename: focal +Architectures: amd64 i386 source +Components: main-focal +Description: PostgresPro pg_probackup repo +SignWith: yes +Log: + --type=dsc changelog.script diff --git a/packaging/repo/rpm-conf/rpmmacros b/packaging/repo/rpm-conf/rpmmacros new file mode 100644 index 000000000..e00a76d68 --- /dev/null +++ b/packaging/repo/rpm-conf/rpmmacros @@ -0,0 +1,5 @@ +%_signature gpg +%_gpg_path /root/.gnupg +%_gpg_name PostgreSQL Professional +%_gpgbin /usr/bin/gpg +%_gpg_check_password_cmd /bin/true diff --git a/packaging/repo/scripts/alt.sh b/packaging/repo/scripts/alt.sh new file mode 100755 index 000000000..4cda313ef --- /dev/null +++ b/packaging/repo/scripts/alt.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -exu +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} + +apt-get update -y +apt-get install -qq -y apt-repo-tools gnupg rsync perl less wget + +if [[ ${PBK_EDITION} == '' ]] ; then + REPO_SUFFIX='vanilla' + FORK='PostgreSQL' +else + REPO_SUFFIX='forks' + FORK='PostgresPro' +fi + +cd $INPUT_DIR + +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg +for pkg in $(ls); do + for pkg_full_version in $(ls ./$pkg); do + + # THere is no std/ent packages for PG 9.5 + if [[ ${pkg} == 'pg_probackup-std-9.5' ]] || [[ ${pkg} == 'pg_probackup-ent-9.5' ]] ; then + continue; + fi + + RPM_DIR=${OUT_DIR}/rpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION}/x86_64/RPMS.${REPO_SUFFIX} + mkdir -p "$RPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + + genbasedir --architecture=x86_64 --architectures=x86_64 --origin=repo.postgrespro.ru \ + --label="${FORK} backup utility pg_probackup" --description "${FORK} pg_probackup repo" \ + --version=$pkg_full_version --bloat --progress --create \ + --topdir=${OUT_DIR}/rpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION} x86_64 ${REPO_SUFFIX} + + # SRPM is available only for vanilla + if [[ ${PBK_EDITION} == '' ]] ; then + SRPM_DIR=${OUT_DIR}/srpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION}/x86_64/SRPMS.${REPO_SUFFIX} + mkdir -p "$SRPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + + genbasedir --architecture=x86_64 --architectures=x86_64 --origin=repo.postgrespro.ru \ + --label="${FORK} backup utility pg_probackup sources" --description "${FORK} pg_probackup repo" \ + --version=$pkg_full_version --bloat --progress --create \ + --topdir=${OUT_DIR}/srpm/${pkg_full_version}/altlinux-p${DISTRIB_VERSION} x86_64 ${REPO_SUFFIX} + fi + done +done diff --git a/packaging/repo/scripts/deb.sh b/packaging/repo/scripts/deb.sh new file mode 100755 index 000000000..31416972d --- /dev/null +++ b/packaging/repo/scripts/deb.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -exu +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in # dir with builded deb +export OUT_DIR=/app/www/${PBK_PKG_REPO} +#export REPO_DIR=/app/repo + +cd $INPUT_DIR + +export DEB_DIR=$OUT_DIR/deb +export KEYS_DIR=$OUT_DIR/keys +export CONF=/app/repo/reprepro-conf +mkdir -p "$KEYS_DIR" +cp -av /app/repo/${PBK_PKG_REPO}/gnupg /root/.gnupg + +rsync /app/repo/${PBK_PKG_REPO}/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +mkdir -p $DEB_DIR +cd $DEB_DIR +cp -av $CONF ./conf + +# make remove-debpkg tool +echo -n "#!" > remove-debpkg +echo "/bin/sh" >> remove-debpkg +echo "CODENAME=\$1" >> remove-debpkg +echo "DEBFILE=\$2" >> remove-debpkg +echo "DEBNAME=\`basename \$DEBFILE | sed -e 's/_.*//g'\`" >> remove-debpkg +echo "reprepro --waitforlock 5 remove \$CODENAME \$DEBNAME" >> remove-debpkg +chmod +x remove-debpkg + +#find $INPUT_DIR/ -name '*.changes' -exec reprepro -P optional -Vb . include ${CODENAME} {} \; +find $INPUT_DIR -name "*${CODENAME}*.deb" -exec ./remove-debpkg $CODENAME {} \; +find $INPUT_DIR -name "*${CODENAME}*.dsc" -exec reprepro --waitforlock 5 -i undefinedtarget --ignore=missingfile -P optional -S main -Vb . includedsc $CODENAME {} \; +find $INPUT_DIR -name "*${CODENAME}*.deb" -exec reprepro --waitforlock 5 -i undefinedtarget --ignore=missingfile -P optional -Vb . includedeb $CODENAME {} \; +reprepro export $CODENAME + +rm -f remove-debpkg +rm -rf ./conf +rm -rf /root/.gnupg diff --git a/packaging/repo/scripts/rpm.sh b/packaging/repo/scripts/rpm.sh new file mode 100755 index 000000000..524789cbf --- /dev/null +++ b/packaging/repo/scripts/rpm.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -ex +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} +export KEYS_DIR=$OUT_DIR/keys + +# deploy keys +mkdir -p "$KEYS_DIR" +rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP +chmod 755 $KEYS_DIR +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +cd $INPUT_DIR + +cp -arv /app/repo/rpm-conf/rpmmacros /root/.rpmmacros +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg +chown -R root:root /root/.gnupg + +for pkg in $(ls ${INPUT_DIR}); do + for pkg_full_version in $(ls ${INPUT_DIR}/$pkg); do + if [[ ${PBK_EDITION} == '' ]] ; then + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-$DISTRIB.noarch.rpm + else + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-forks-$DISTRIB.noarch.rpm + fi + + [ ! -z "$CODENAME" ] && export DISTRIB_VERSION=$CODENAME + RPM_DIR=$OUT_DIR/rpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + mkdir -p "$RPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + for f in $(ls $RPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + createrepo $RPM_DIR/ + + if [[ ${PBK_EDITION} == '' ]] ; then + SRPM_DIR=$OUT_DIR/srpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + mkdir -p "$SRPM_DIR" + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + for f in $(ls $SRPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + createrepo $SRPM_DIR/ + fi + + done +done diff --git a/packaging/repo/scripts/suse.sh b/packaging/repo/scripts/suse.sh new file mode 100755 index 000000000..c85e0ff10 --- /dev/null +++ b/packaging/repo/scripts/suse.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -ex +set -o errexit +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +export INPUT_DIR=/app/in #dir with builded rpm +export OUT_DIR=/app/www/${PBK_PKG_REPO} +export KEYS_DIR=$OUT_DIR/keys +# deploy keys + +zypper install -y createrepo +rm -rf /root/.gnupg + +cd $INPUT_DIR + +mkdir -p $KEYS_DIR +chmod 755 $KEYS_DIR +rsync /app/repo/$PBK_PKG_REPO/gnupg/key.public $KEYS_DIR/GPG-KEY-PG_PROBACKUP + +echo -e 'User-agent: *\nDisallow: /' > $OUT_DIR/robots.txt + +cp -arv /app/repo/rpm-conf/rpmmacros /root/.rpmmacros +cp -arv /app/repo/$PBK_PKG_REPO/gnupg /root/.gnupg +chmod -R 0600 /root/.gnupg + +for pkg in $(ls); do + for pkg_full_version in $(ls ./$pkg); do + + cp $INPUT_DIR/$pkg/$pkg_full_version/RPMS/noarch/pg_probackup-repo-*.noarch.rpm \ + $KEYS_DIR/pg_probackup-repo-$DISTRIB.noarch.rpm + [ ! -z "$CODENAME" ] && export DISTRIB_VERSION=$CODENAME + RPM_DIR=$OUT_DIR/rpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + SRPM_DIR=$OUT_DIR/srpm/$pkg_full_version/${DISTRIB}-${DISTRIB_VERSION}-x86_64 + + # rm -rf "$RPM_DIR" && mkdir -p "$RPM_DIR" + # rm -rf "$SRPM_DIR" && mkdir -p "$SRPM_DIR" + mkdir -p "$RPM_DIR" + mkdir -p "$SRPM_DIR" + + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/RPMS/x86_64/* $RPM_DIR/ + cp -arv $INPUT_DIR/$pkg/$pkg_full_version/SRPMS/* $SRPM_DIR/ + + for f in $(ls $RPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + for f in $(ls $SRPM_DIR/*.rpm); do rpm --addsign $f || exit 1; done + + createrepo $RPM_DIR/ + createrepo $SRPM_DIR/ + + # rpm --addsign $RPM_DIR/repodata/repomd.xml + # rpm --addsign $SRPM_DIR/repodata/repomd.xml + + gpg --batch --yes -a --detach-sign $RPM_DIR/repodata/repomd.xml + gpg --batch --yes -a --detach-sign $SRPM_DIR/repodata/repomd.xml + + cp -a /root/.gnupg/key.public $RPM_DIR/repodata/repomd.xml.key + cp -a /root/.gnupg/key.public $SRPM_DIR/repodata/repomd.xml.key + done +done diff --git a/packaging/test/Makefile.alt b/packaging/test/Makefile.alt new file mode 100644 index 000000000..bd00cef7f --- /dev/null +++ b/packaging/test/Makefile.alt @@ -0,0 +1,50 @@ +# ALT 9 +build/test_alt_9_9.6: + $(call test_alt,alt,9,,9.6,9.6.24) + touch build/test_alt_9_9.6 + +build/test_alt_9_10: + $(call test_alt,alt,9,,10,10.19) + touch build/test_alt_9_10 + +build/test_alt_9_11: + $(call test_alt,alt,9,,11,11.14) + touch build/test_alt_9_11 + +build/test_alt_9_12: + $(call test_alt,alt,9,,12,12.9) + touch build/test_alt_9_12 + +build/test_alt_9_13: + $(call test_alt,alt,9,,13,13.5) + touch build/test_alt_9_13 + +build/test_alt_9_14: + $(call test_alt,alt,9,,14,14.1) + touch build/test_alt_9_14 + +# ALT 8 +build/test_alt_8_9.6: + $(call test_alt,alt,8,,9.6,9.6.24) + touch build/test_alt_8_9.6 + +build/test_alt_8_10: + $(call test_alt,alt,8,,10,10.19) + touch build/test_alt_8_10 + +build/test_alt_8_11: + $(call test_alt,alt,8,,11,11.14) + touch build/test_alt_8_11 + +build/test_alt_8_12: + $(call test_alt,alt,8,,12,12.9) + touch build/test_alt_8_12 + +build/test_alt_8_13: + $(call test_alt,alt,8,,13,13.5) + touch build/test_alt_8_13 + +build/test_alt_8_14: + $(call test_alt,alt,8,,14,14.1) + touch build/test_alt_8_14 + diff --git a/packaging/test/Makefile.centos b/packaging/test/Makefile.centos new file mode 100644 index 000000000..9d30a324b --- /dev/null +++ b/packaging/test/Makefile.centos @@ -0,0 +1,49 @@ +# CENTOS 7 +build/test_centos_7_9.6: + $(call test_rpm,centos,7,,9.6,9.6.24) + touch build/test_centos_7_9.6 + +build/test_centos_7_10: + $(call test_rpm,centos,7,,10,10.19) + touch build/test_centos_7_10 + +build/test_centos_7_11: + $(call test_rpm,centos,7,,11,11.14) + touch build/test_centos_7_11 + +build/test_centos_7_12: + $(call test_rpm,centos,7,,12,12.9) + touch build/test_centos_7_12 + +build/test_centos_7_13: + $(call test_rpm,centos,7,,13,13.5) + touch build/test_centos_7_13 + +build/test_centos_7_14: + $(call test_rpm,centos,7,,14,14.1) + touch build/test_centos_7_14 + +# CENTOS 8 +build/test_centos_8_9.6: + $(call test_rpm,centos,8,,9.6,9.6.24) + touch build/test_centos_8_9.6 + +build/test_centos_8_10: + $(call test_rpm,centos,8,,10,10.19) + touch build/test_centos_8_10 + +build/test_centos_8_11: + $(call test_rpm,centos,8,,11,11.14) + touch build/test_centos_8_11 + +build/test_centos_8_12: + $(call test_rpm,centos,8,,12,12.9) + touch build/test_centos_8_12 + +build/test_centos_8_13: + $(call test_rpm,centos,8,,13,13.5) + touch build/test_centos_8_13 + +build/test_centos_8_14: + $(call test_rpm,centos,8,,14,14.1) + touch build/test_centos_8_14 diff --git a/packaging/test/Makefile.debian b/packaging/test/Makefile.debian new file mode 100644 index 000000000..e4d904f62 --- /dev/null +++ b/packaging/test/Makefile.debian @@ -0,0 +1,74 @@ +# DEBIAN 9 +build/test_debian_9_9.6: + $(call test_deb,debian,9,stretch,9.6,9.6.24) + touch build/test_debian_9_9.6 + +build/test_debian_9_10: + $(call test_deb,debian,9,stretch,10,10.19) + touch build/test_debian_9_10 + +build/test_debian_9_11: + $(call test_deb,debian,9,stretch,11,11.14) + touch build/test_debian_9_11 + +build/test_debian_9_12: + $(call test_deb,debian,9,stretch,12,12.9) + touch build/test_debian_9_12 + +build/test_debian_9_13: + $(call test_deb,debian,9,stretch,13,13.5) + touch build/test_debian_9_13 + +build/test_debian_9_14: + $(call test_deb,debian,9,stretch,14,14.1) + touch build/test_debian_9_14 + +# DEBIAN 10 +build/test_debian_10_9.6: + $(call test_deb,debian,10,buster,9.6,9.6.24) + touch build/test_debian_10_9.6 + +build/test_debian_10_10: + $(call test_deb,debian,10,buster,10,10.19) + touch build/test_debian_10_10 + +build/test_debian_10_11: + $(call test_deb,debian,10,buster,11,11.14) + touch build/test_debian_10_11 + +build/test_debian_10_12: + $(call test_deb,debian,10,buster,12,12.9) + touch build/test_debian_10_12 + +build/test_debian_10_13: + $(call test_deb,debian,10,buster,13,13.5) + touch build/test_debian_10_13 + +build/test_debian_10_14: + $(call test_deb,debian,10,buster,14,14.1) + touch build/test_debian_10_14 + +# DEBIAN 11 +build/test_debian_11_9.6: + $(call test_deb,debian,11,bullseye,9.6,9.6.24) + touch build/test_debian_11_9.6 + +build/test_debian_11_10: + $(call test_deb,debian,11,bullseye,10,10.19) + touch build/test_debian_11_10 + +build/test_debian_11_11: + $(call test_deb,debian,11,bullseye,11,11.14) + touch build/test_debian_11_11 + +build/test_debian_11_12: + $(call test_deb,debian,11,bullseye,12,12.9) + touch build/test_debian_11_12 + +build/test_debian_11_13: + $(call test_deb,debian,11,bullseye,13,13.5) + touch build/test_debian_11_13 + +build/test_debian_11_14: + $(call test_deb,debian,11,bullseye,14,14.1) + touch build/test_debian_11_14 diff --git a/packaging/test/Makefile.oraclelinux b/packaging/test/Makefile.oraclelinux new file mode 100644 index 000000000..0efe6574d --- /dev/null +++ b/packaging/test/Makefile.oraclelinux @@ -0,0 +1,49 @@ +# ORACLE LINUX 7 +build/test_oraclelinux_7_9.6: + $(call test_rpm,oraclelinux,7,,9.6,9.6.24) + touch build/test_oraclelinux_7_9.6 + +build/test_oraclelinux_7_10: + $(call test_rpm,oraclelinux,7,,10,10.19) + touch build/test_oraclelinux_7_10 + +build/test_oraclelinux_7_11: + $(call test_rpm,oraclelinux,7,,11,11.14) + touch build/test_oraclelinux_7_11 + +build/test_oraclelinux_7_12: + $(call test_rpm,oraclelinux,7,,12,12.9) + touch build/test_oraclelinux_7_12 + +build/test_oraclelinux_7_13: + $(call test_rpm,oraclelinux,7,,13,13.5) + touch build/test_oraclelinux_7_13 + +build/test_oraclelinux_7_14: + $(call test_rpm,oraclelinux,7,,14,14.1) + touch build/test_oraclelinux_7_14 + +# ORACLE LINUX 8 +build/test_oraclelinux_8_9.6: + $(call test_rpm,oraclelinux,8,,9.6,9.6.24) + touch build/test_oraclelinux_8_9.6 + +build/test_oraclelinux_8_10: + $(call test_rpm,oraclelinux,8,,10,10.19) + touch build/test_oraclelinux_8_10 + +build/test_oraclelinux_8_11: + $(call test_rpm,oraclelinux,8,,11,11.14) + touch build/test_oraclelinux_8_11 + +build/test_oraclelinux_8_12: + $(call test_rpm,oraclelinux,8,,12,12.9) + touch build/test_oraclelinux_8_12 + +build/test_oraclelinux_8_13: + $(call test_rpm,oraclelinux,8,,13,13.5) + touch build/test_oraclelinux_8_13 + +build/test_oraclelinux_8_14: + $(call test_rpm,oraclelinux,8,,14,14.1) + touch build/test_oraclelinux_8_14 diff --git a/packaging/test/Makefile.rhel b/packaging/test/Makefile.rhel new file mode 100644 index 000000000..3b26c8942 --- /dev/null +++ b/packaging/test/Makefile.rhel @@ -0,0 +1,49 @@ +# RHEL 7 +build/test_rhel_7_9.6: + $(call test_rpm,rhel,7,7Server,9.6,9.6.24) + touch build/test_rhel_7_9.6 + +build/test_rhel_7_10: + $(call test_rpm,rhel,7,7Server,10,10.19) + touch build/test_rhel_7_10 + +build/test_rhel_7_11: + $(call test_rpm,rhel,7,7Server,11,11.14) + touch build/test_rhel_7_11 + +build/test_rhel_7_12: + $(call test_rpm,rhel,7,7Server,12,12.9) + touch build/test_rhel_7_12 + +build/test_rhel_7_13: + $(call test_rpm,rhel,7,7Server,13,13.5) + touch build/test_rhel_7_13 + +build/test_rhel_7_14: + $(call test_rpm,rhel,7,7Server,14,14.1) + touch build/test_rhel_7_14 + +# RHEL 8 +build/test_rhel_8_9.6: + $(call test_rpm,rhel,8,8Server,9.6,9.6.24) + touch build/test_rhel_8_9.6 + +build/test_rhel_8_10: + $(call test_rpm,rhel,8,8Server,10,10.19) + touch build/test_rhel_8_10 + +build/test_rhel_8_11: + $(call test_rpm,rhel,8,8Server,11,11.14) + touch build/test_rhel_8_11 + +build/test_rhel_8_12: + $(call test_rpm,rhel,8,8Server,12,12.9) + touch build/test_rhel_8_12 + +build/test_rhel_8_13: + $(call test_rpm,rhel,8,8Server,13,13.5) + touch build/test_rhel_8_13 + +build/test_rhel_8_14: + $(call test_rpm,rhel,8,8Server,14,14.1) + touch build/test_rhel_8_14 diff --git a/packaging/test/Makefile.suse b/packaging/test/Makefile.suse new file mode 100644 index 000000000..19e8d52d8 --- /dev/null +++ b/packaging/test/Makefile.suse @@ -0,0 +1,49 @@ +# Suse 15.1 +build/test_suse_15.1_9.6: + $(call test_suse,suse,15.1,,9.6,9.6.24) + touch build/test_suse_15.1_9.6 + +build/test_suse_15.1_10: + $(call test_suse,suse,15.1,,10,10.19) + touch build/test_suse_15.1_10 + +build/test_suse_15.1_11: + $(call test_suse,suse,15.1,,11,11.14) + touch build/test_suse_15.1_11 + +build/test_suse_15.1_12: + $(call test_suse,suse,15.1,,12,12.9) + touch build/test_suse_15.1_12 + +build/test_suse_15.1_13: + $(call test_suse,suse,15.1,,13,13.5) + touch build/test_suse_15.1_13 + +build/test_suse_15.1_14: + $(call test_suse,suse,15.1,,14,14.1) + touch build/test_suse_15.1_14 + +# Suse 15.2 +build/test_suse_15.2_9.6: + $(call test_suse,suse,15.2,,9.6,9.6.24) + touch build/test_suse_15.2_9.6 + +build/test_suse_15.2_10: + $(call test_suse,suse,15.2,,10,10.19) + touch build/test_suse_15.2_10 + +build/test_suse_15.2_11: + $(call test_suse,suse,15.2,,11,11.14) + touch build/test_suse_15.2_11 + +build/test_suse_15.2_12: + $(call test_suse,suse,15.2,,12,12.9) + touch build/test_suse_15.2_12 + +build/test_suse_15.2_13: + $(call test_suse,suse,15.2,,13,13.5) + touch build/test_suse_15.2_13 + +build/test_suse_15.2_14: + $(call test_suse,suse,15.2,,14,14.1) + touch build/test_suse_15.2_14 diff --git a/packaging/test/Makefile.ubuntu b/packaging/test/Makefile.ubuntu new file mode 100644 index 000000000..86a257b91 --- /dev/null +++ b/packaging/test/Makefile.ubuntu @@ -0,0 +1,49 @@ +# UBUNTU 18.04 +build/test_ubuntu_18.04_9.6: + $(call test_deb,ubuntu,18.04,bionic,9.6,9.6.24) + touch build/test_ubuntu_18.04_9.6 + +build/test_ubuntu_18.04_10: + $(call test_deb,ubuntu,18.04,bionic,10,10.19) + touch build/test_ubuntu_18.04_10 + +build/test_ubuntu_18.04_11: + $(call test_deb,ubuntu,18.04,bionic,11,11.14) + touch build/test_ubuntu_18.04_11 + +build/test_ubuntu_18.04_12: + $(call test_deb,ubuntu,18.04,bionic,12,12.9) + touch build/test_ubuntu_18.04_12 + +build/test_ubuntu_18.04_13: + $(call test_deb,ubuntu,18.04,bionic,13,13.5) + touch build/test_ubuntu_18.04_13 + +build/test_ubuntu_18.04_14: + $(call test_deb,ubuntu,18.04,bionic,14,14.1) + touch build/test_ubuntu_18.04_14 + +# UBUNTU 20.04 +build/test_ubuntu_20.04_9.6: + $(call test_deb,ubuntu,20.04,focal,9.6,9.6.24) + touch build/test_ubuntu_20.04_9.6 + +build/test_ubuntu_20.04_10: + $(call test_deb,ubuntu,20.04,focal,10,10.19) + touch build/test_ubuntu_20.04_10 + +build/test_ubuntu_20.04_11: + $(call test_deb,ubuntu,20.04,focal,11,11.14) + touch build/test_ubuntu_20.04_11 + +build/test_ubuntu_20.04_12: + $(call test_deb,ubuntu,20.04,focal,12,12.9) + touch build/test_ubuntu_20.04_12 + +build/test_ubuntu_20.04_13: + $(call test_deb,ubuntu,20.04,focal,13,13.5) + touch build/test_ubuntu_20.04_13 + +build/test_ubuntu_20.04_14: + $(call test_deb,ubuntu,20.04,focal,14,14.1) + touch build/test_ubuntu_20.04_14 diff --git a/packaging/test/scripts/alt.sh b/packaging/test/scripts/alt.sh new file mode 100755 index 000000000..262864474 --- /dev/null +++ b/packaging/test/scripts/alt.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -xe +set -o pipefail + +ulimit -n 1024 + +apt-get clean -y +apt-get update -y +apt-get install nginx su -y + +adduser nginx + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF + +/etc/init.d/nginx start + +# install POSTGRESQL + +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p7 x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +apt-get update +apt-get install ${PKG_NAME} -y +${PKG_NAME} --help +${PKG_NAME} --version + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +echo "rpm http://repo.postgrespro.ru/pg_probackup/rpm/latest/altlinux-p${DISTRIB_VERSION} x86_64 vanilla" > /etc/apt/sources.list.d/pg_probackup.list +echo "rpm [p${DISTRIB_VERSION}] http://mirror.yandex.ru/altlinux p${DISTRIB_VERSION}/branch/x86_64 debuginfo" > /etc/apt/sources.list.d/debug.list + +apt-get update -y +apt-get install ${PKG_NAME} -y +${PKG_NAME} --help +${PKG_NAME} --version + +exit 0 + +# TODO: run init, add-instance, backup and restore +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync --compress" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt diff --git a/packaging/test/scripts/alt_forks.sh b/packaging/test/scripts/alt_forks.sh new file mode 100755 index 000000000..c406e5358 --- /dev/null +++ b/packaging/test/scripts/alt_forks.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +ulimit -n 1024 + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +apt-get clean -y +apt-get update -y +apt-get install nginx su -y + +adduser nginx + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF + +/etc/init.d/nginx start + +# install POSTGRESQL + +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +echo "rpm http://repo.postgrespro.ru/pg_probackup-forks/rpm/latest/altlinux-p${DISTRIB_VERSION} x86_64 forks" > /etc/apt/sources.list.d/pg_probackup.list +echo "rpm [p${DISTRIB_VERSION}] http://mirror.yandex.ru/altlinux p${DISTRIB_VERSION}/branch/x86_64 debuginfo" > /etc/apt/sources.list.d/debug.list + +apt-get update -y +apt-get install ${PKG_NAME} ${PKG_NAME}-debuginfo -y +${PKG_NAME} --help +${PKG_NAME} --version + +exit 0 + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt diff --git a/packaging/test/scripts/deb.sh b/packaging/test/scripts/deb.sh new file mode 100755 index 000000000..fca9a23d8 --- /dev/null +++ b/packaging/test/scripts/deb.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +# upgrade and utils +# export parameters +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +if [ ${DISTRIB} = 'ubuntu' -a ${CODENAME} = 'xenial' ] ; then + apt-get -qq update +elif [ ${DISTRIB} = 'debian' -a ${CODENAME} = 'stretch' ] ; then + apt-get -qq update +else + apt-get -qq --allow-releaseinfo-change update +fi + +apt-get -qq install -y wget nginx gnupg lsb-release +#apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps + +# echo -e 'Package: *\nPin: origin test.postgrespro.ru\nPin-Priority: 800' >\ +# /etc/apt/preferences.d/pgpro-800 + +# install nginx +echo "127.0.0.1 test.postgrespro.ru" >> /etc/hosts +cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --no-check-certificate -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + apt-get update -y + apt-get install -y postgresql-${PG_VERSION} +#fi + +# install pg_probackup from current public repo +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-old.list +wget -O - http://repo.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get install -y pg-probackup-${PG_VERSION} +pg_probackup-${PG_VERSION} --help +pg_probackup-${PG_VERSION} --version + +# Artful do no have PostgreSQL packages at all, Precise do not have PostgreSQL 10 +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + export PGDATA=/var/lib/postgresql/${PG_VERSION}/data + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/initdb -k -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} init -B /tmp/backup" + su postgres -c "pg_probackup-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='pg_probackup-${PG_VERSION} archive-push --no-sync -B /tmp/backup compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" + sleep 5 + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b full -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup --archive -D ${PGDATA}" + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pgbench --no-vacuum -i -s 5" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +#fi + +# install new packages +echo "deb [arch=amd64] http://test.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-new.list +wget -O - http://test.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - +apt-get update +apt-get install -y pg-probackup-${PG_VERSION} +pg_probackup-${PG_VERSION} --help +pg_probackup-${PG_VERSION} --version + +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl restart -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + +# su postgres -c "pg_probackup-${PG_VERSION} init -B /tmp/backup" +# su postgres -c "pg_probackup-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node --compress -b delta -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" + su postgres -c "pg_probackup-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" + rm -rf ${PGDATA} + + su postgres -c "pg_probackup-${PG_VERSION} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/pg_ctl start -w -t 60 -D ${PGDATA}" + +sleep 5 +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/lib/postgresql/${PG_VERSION}/bin/psql" || exit 1 +#fi + +# CHECK SRC package +apt-get install -y dpkg-dev +echo "deb-src [arch=amd64] http://test.postgrespro.ru/pg_probackup/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >>\ + /etc/apt/sources.list.d/pg_probackup.list + +wget -O - http://test.postgrespro.ru/pg_probackup/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get update -y + +cd /mnt +apt-get source pg-probackup-${PG_VERSION} +exit 0 + +cd pg-probackup-${PG_VERSION}-${PKG_VERSION} +#mk-build-deps --install --remove --tool 'apt-get --no-install-recommends --yes' debian/control +#rm -rf ./*.deb +apt-get install -y debhelper bison flex gettext zlib1g-dev +dpkg-buildpackage -us -uc diff --git a/packaging/test/scripts/deb_forks.sh b/packaging/test/scripts/deb_forks.sh new file mode 100755 index 000000000..e05695608 --- /dev/null +++ b/packaging/test/scripts/deb_forks.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# TODO: remove after release +exit 0 + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then + exit 0 +fi + +# upgrade and utils +# export parameters +export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +#if [ ${CODENAME} == 'jessie' ]; then +#printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list +#fi + +if [ ${DISTRIB} = 'debian' -a ${CODENAME} = 'stretch' ] ; then + apt-get -qq update +else + apt-get -qq --allow-releaseinfo-change update +fi +apt-get -qq install -y wget nginx gnupg lsb-release apt-transport-https +#apt-get -qq install -y libterm-readline-gnu-perl dialog gnupg procps + +# echo -e 'Package: *\nPin: origin test.postgrespro.ru\nPin-Priority: 800' >\ +# /etc/apt/preferences.d/pgpro-800 + +# install nginx +echo "127.0.0.1 test.postgrespro.ru" >> /etc/hosts +cat < /etc/nginx/nginx.conf +user www-data; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESPRO +if [ ${PBK_EDITION} == 'std' ]; then + sh -c 'echo "deb https://repo.postgrespro.ru/pgpro-${PG_VERSION}/${DISTRIB}/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/pgpro.list' + wget --quiet -O - https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/GPG-KEY-POSTGRESPRO | apt-key add - + apt-get update -y + + apt-get install -y postgrespro-std-${PG_VERSION} + BINDIR="/opt/pgpro/std-${PG_VERSION}/bin" + export LD_LIBRARY_PATH=/opt/pgpro/std-${PG_VERSION}/lib/ +fi + +# install pg_probackup from current public repo +echo "deb [arch=amd64] http://repo.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-old.list +wget -O - http://repo.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | apt-key add - && apt-get update + +apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} +pg_probackup-${PBK_EDITION}-${PG_VERSION} --help +pg_probackup-${PBK_EDITION}-${PG_VERSION} --version + + +if [ ${PBK_EDITION} == 'std' ]; then + export PGDATA=/tmp/data + su postgres -c "${BINDIR}/initdb -k -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} init -B /tmp/backup" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='pg_probackup-${PBK_EDITION}-${PG_VERSION} archive-push --no-sync -B /tmp/backup compress --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + + su postgres -c "${BINDIR}/pg_ctl stop -w -t 60 -D /var/lib/pgpro/std-${PG_VERSION}/data" || echo "it is all good" + su postgres -c "${BINDIR}/pg_ctl start -D ${PGDATA}" + sleep 5 + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b full -B /tmp/backup -D ${PGDATA} --no-sync" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + + su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +fi + +# install new packages +echo "deb [arch=amd64] http://test.postgrespro.ru/pg_probackup-forks/deb/ $(lsb_release -cs) main-$(lsb_release -cs)" >\ + /etc/apt/sources.list.d/pg_probackup-new.list +wget -O - http://test.postgrespro.ru/pg_probackup-forks/keys/GPG-KEY-PG_PROBACKUP | apt-key add - +apt-get update -y + +#if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then +# apt-get install -y libpq5 pg-probackup-${PBK_EDITION}-${PG_VERSION} +#else +# apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} +#fi + +apt-get install -y pg-probackup-${PBK_EDITION}-${PG_VERSION} + +# in Ent 11 and 10 because of PQselect vanilla libpq5 is incompatible with Ent pg_probackup +if [ ${PBK_EDITION} == 'ent' ]; then + if [ ${PG_VERSION} == '11' ] || [ ${PG_VERSION} == '10' ] || [ ${PG_VERSION} == '9.6' ]; then + exit 0 + fi +fi + +pg_probackup-${PBK_EDITION}-${PG_VERSION} --help +pg_probackup-${PBK_EDITION}-${PG_VERSION} --version + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +if [ ${PBK_EDITION} == 'std' ] && [ ${PG_VERSION} == '9.6' ]; then + exit 0 +fi + + +#if [ ${CODENAME} == 'precise' ] && [ ${PG_VERSION} != '10' ] && [ ${PG_VERSION} != '11' ]; then + su postgres -c "${BINDIR}/pgbench --no-vacuum -t 1000 -c 1" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} show --instance=node -B /tmp/backup -D ${PGDATA}" + + su postgres -c "${BINDIR}/pg_ctl stop -w -t 60 -D ${PGDATA}" + rm -rf ${PGDATA} + + su postgres -c "pg_probackup-${PBK_EDITION}-${PG_VERSION} restore --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "${BINDIR}/pg_ctl start -w -t 60 -D ${PGDATA}" + +sleep 5 +echo "select count(*) from pgbench_accounts;" | su postgres -c "${BINDIR}/psql" || exit 1 + +exit 0 diff --git a/packaging/test/scripts/rpm.sh b/packaging/test/scripts/rpm.sh new file mode 100755 index 000000000..87d430ef8 --- /dev/null +++ b/packaging/test/scripts/rpm.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2021 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then + # update of rpm package is broken in rhel-7 (26/12/2022) + #yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi + yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi +fi +# yum upgrade -y || echo 'some packages in docker failed to upgrade' +# yum install -y sudo + +if [ ${DISTRIB_VERSION} = '6' ]; then + yum install -y https://nginx.org/packages/rhel/6/x86_64/RPMS/nginx-1.8.1-1.el6.ngx.x86_64.rpm +elif [ ${DISTRIB_VERSION} = '7' ]; then + yum install -y https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.8.1-1.el7.ngx.x86_64.rpm +elif [ ${DISTRIB_VERSION} = '8' -a \( ${DISTRIB} = 'rhel' -o ${DISTRIB} = 'oraclelinux' \) ]; then + yum install -y nginx +else + yum install epel-release -y + yum install -y nginx +fi + +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-${DISTRIB_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm + +if [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then + dnf -qy module disable postgresql +fi + +if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then + dnf -qy module disable postgresql +fi + +# PGDG doesn't support install of PG-9.6 from repo package anymore +if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then + # ugly hack: use repo settings from PG10 + sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo +fi + +yum install -y postgresql${PG_TOG}-server.x86_64 +export PGDATA=/var/lib/pgsql/${PG_VERSION}/data + +# install old packages +yum install -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +yum install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf +echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +echo "archive_command='${PKG_NAME} archive-push --no-sync -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +sleep 5 + +su postgres -c "${PKG_NAME} init -B /tmp/backup" +su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts + +# yum remove -y pg_probackup-repo +#yum install -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +yum clean all -y + +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup.repo + +yum update -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 + +#else +# echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +# rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +# yum install -y ${PKG_NAME} +# ${PKG_NAME} --help +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 10" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +# rm -rf ${PGDATA} +# su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 10 +# echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 +#fi + +exit 0 # while PG12 is not working + +# SRC PACKAGE +cd /mnt +yum install yum-utils rpm-build -y +yumdownloader --source ${PKG_NAME} +rpm -ivh ./*.rpm +cd /root/rpmbuild/SPECS +exit 0 + +# build pg_probackup +yum-builddep -y pg_probackup.spec +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log diff --git a/packaging/test/scripts/rpm_forks.sh b/packaging/test/scripts/rpm_forks.sh new file mode 100755 index 000000000..d57711697 --- /dev/null +++ b/packaging/test/scripts/rpm_forks.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +if [ ${DISTRIB} != 'rhel' -o ${DISTRIB_VERSION} != '7' ]; then + # update of rpm package is broken in rhel-7 (26/12/2022) + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi + yum update -y + if [ ${DISTRIB} = 'centos' -a ${DISTRIB_VERSION} = '8' ]; then + sed -i 's|mirrorlist|#mirrorlist|g' /etc/yum.repos.d/CentOS-*.repo + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*.repo + fi +fi + +if [ ${PBK_EDITION} == 'ent' ]; then + exit 0 +fi + +# yum upgrade -y || echo 'some packages in docker failed to upgrade' +# yum install -y sudo + +if [ ${DISTRIB} == 'rhel' ] && [ ${DISTRIB_VERSION} == '6' ]; then + exit 0; +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ]; then + exit 0; +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '8' ]; then + yum install -y nginx +elif [ ${DISTRIB_VERSION} == '7' ]; then + yum install -y https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.8.1-1.el7.ngx.x86_64.rpm +elif [ ${DISTRIB} == 'oraclelinux' ] && [ ${DISTRIB_VERSION} == '6' ]; then + yum install -y https://nginx.org/packages/rhel/6/x86_64/RPMS/nginx-1.8.1-1.el6.ngx.x86_64.rpm +else + yum install epel-release -y + yum install -y nginx +fi + +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# if [ ${DISTRIB} == 'centos' ]; then + +# install old packages +yum install -y http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-${DISTRIB}.noarch.rpm +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup-forks.repo + +yum install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +if [ $PBK_EDITION == 'std' ] ; then + + # install POSTGRESQL + # rpm -ivh https://download.postgresql.org/pub/repos/yum/reporpms/EL-${DISTRIB_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm + #if [[ ${PG_VERSION} == '11' ]] || [[ ${PG_VERSION} == '12' ]]; then + # rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + #else + # rpm -ivh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/postgrespro-std-${PG_VERSION}.${DISTRIB}.yum-${PG_VERSION}-0.3.noarch.rpm + #fi + curl -o pgpro-repo-add.sh https://repo.postgrespro.ru/pgpro-${PG_VERSION}/keys/pgpro-repo-add.sh + sh pgpro-repo-add.sh + + if [[ ${PG_VERSION} == '9.6' ]]; then + yum install -y postgrespro${PG_TOG}-server.x86_64 + BINDIR="/usr/pgpro-${PG_VERSION}/bin" + else + yum install -y postgrespro-std-${PG_TOG}-server.x86_64 + BINDIR="/opt/pgpro/std-${PG_VERSION}/bin" + export LD_LIBRARY_PATH=/opt/pgpro/std-${PG_VERSION}/lib/ + fi + + export PGDATA=/tmp/data + + su postgres -c "${BINDIR}/initdb -k -D ${PGDATA}" + echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf + echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf + echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf + echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf + su postgres -c "${BINDIR}/pg_ctl start -D ${PGDATA}" + sleep 5 + + su postgres -c "${PKG_NAME} init -B /tmp/backup" + su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" + su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" + su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + + su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" + su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +fi + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts + +# yum remove -y pg_probackup-repo +#yum install -y http://repo.postgrespro.ru/pg_probackup-forks/keys/pg_probackup-repo-forks-${DISTRIB}.noarch.rpm +#yum clean all -y + +sed -i "s/https/http/g" /etc/yum.repos.d/pg_probackup-forks.repo + +# yum update -y ${PKG_NAME} +yum install -y ${PKG_NAME} + +${PKG_NAME} --help +${PKG_NAME} --version + +if [ $PBK_EDITION == 'ent' ]; then + exit 0 +fi + +# su postgres -c "${BINDIR}/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${BINDIR}/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b full -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "${BINDIR}/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${BINDIR}/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "${BINDIR}/psql" || exit 1 + +#else +# echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +# rpm -ivh http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +# yum install -y ${PKG_NAME} +# ${PKG_NAME} --help +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/initdb -k -D ${PGDATA}" +# su postgres -c "${PKG_NAME} init -B /tmp/backup" +# su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +# echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +# echo "archive_command='${PKG_NAME} archive-push -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 5 +# su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -i -s 10" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pgbench --no-vacuum -t 1000 -c 1" +# su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA}" +# su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl stop -D ${PGDATA}" +# rm -rf ${PGDATA} +# su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA}" +# su postgres -c "/usr/pgsql-${PG_VERSION}/bin/pg_ctl start -D ${PGDATA}" +# sleep 10 +# echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/pgsql-${PG_VERSION}/bin/psql" || exit 1 +#fi + +exit 0 diff --git a/packaging/test/scripts/suse.sh b/packaging/test/scripts/suse.sh new file mode 100755 index 000000000..ff630b479 --- /dev/null +++ b/packaging/test/scripts/suse.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# Copyright Notice: +# © (C) Postgres Professional 2015-2016 http://www.postgrespro.ru/ +# Distributed under Apache License 2.0 +# Распространяется по лицензии Apache 2.0 + +set -xe +set -o pipefail + +# fix https://github.com/moby/moby/issues/23137 +ulimit -n 1024 + +# currenctly we do not build std|ent packages for Suse +if [[ ${PBK_EDITION} != '' ]] ; then + exit 0 +fi + +PG_TOG=$(echo $PG_VERSION | sed 's|\.||g') + +if [ ${PG_TOG} == '13' ]; then # no packages for PG13 + exit 0 +fi + +if [ ${PG_TOG} == '11' ]; then # no packages for PG11 + exit 0 +fi + +if [ ${PG_TOG} == '95' ]; then # no packages for PG95 + exit 0 +fi + +zypper install -y nginx +if ! getent group nginx > /dev/null 2>&1 ; then + addgroup --system --quiet nginx +fi +if ! getent passwd nginx > /dev/null 2>&1 ; then + adduser --quiet \ + --system --disabled-login --ingroup nginx \ + --home /var/run/nginx/ --no-create-home \ + nginx +fi + +useradd postgres + +cat < /etc/nginx/nginx.conf +user nginx; +worker_processes 1; +error_log /var/log/nginx/error.log; +events { + worker_connections 1024; +} +http { + server { + listen 80 default; + root /app/www; + } +} +EOF +nginx -s reload || (pkill -9 nginx || nginx -c /etc/nginx/nginx.conf &) + +# install POSTGRESQL +zypper install -y postgresql${PG_TOG} postgresql${PG_TOG}-server postgresql${PG_TOG}-contrib +export PGDATA=/tmp/data + +# install old packages +zypper install --allow-unsigned-rpm -y http://repo.postgrespro.ru/pg_probackup/keys/pg_probackup-repo-${DISTRIB}.noarch.rpm +zypper --gpg-auto-import-keys install -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/initdb -k -D ${PGDATA}" +echo "fsync=off" >> ${PGDATA}/postgresql.auto.conf +echo "wal_level=hot_standby" >> ${PGDATA}/postgresql.auto.conf +echo "archive_mode=on" >> ${PGDATA}/postgresql.auto.conf +echo "archive_command='${PKG_NAME} archive-push --no-sync -B /tmp/backup --instance=node --wal-file-path %p --wal-file-name %f'" >> ${PGDATA}/postgresql.auto.conf +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl start -D ${PGDATA}" +sleep 5 + +su postgres -c "${PKG_NAME} init -B /tmp/backup" +su postgres -c "${PKG_NAME} add-instance --instance=node -B /tmp/backup -D ${PGDATA}" +su postgres -c "${PKG_NAME} backup --instance=node --compress -b full -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA} --archive" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -i -s 5" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +# install new packages +echo "127.0.0.1 repo.postgrespro.ru" >> /etc/hosts +zypper clean all -y + +sed -i "s/https/http/g" /etc/zypp/repos.d/pg_probackup.repo + +zypper update -y ${PKG_NAME} +${PKG_NAME} --help +${PKG_NAME} --version + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pgbench --no-vacuum -t 1000 -c 1" +su postgres -c "${PKG_NAME} backup --instance=node -b page -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "${PKG_NAME} show --instance=node -B /tmp/backup -D ${PGDATA}" + +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl stop -D ${PGDATA}" +rm -rf ${PGDATA} + +su postgres -c "${PKG_NAME} restore --instance=node -B /tmp/backup -D ${PGDATA} --no-sync" +su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/pg_ctl start -w -D ${PGDATA}" + +sleep 5 + +echo "select count(*) from pgbench_accounts;" | su postgres -c "/usr/lib/postgresql${PG_TOG}/bin/psql" || exit 1 + +exit 0 + +# SRC PACKAGE +cd /mnt +yum install yum-utils rpm-build -y +yumdownloader --source ${PKG_NAME} +rpm -ivh ./*.rpm +cd /root/rpmbuild/SPECS +exit 0 + +# build pg_probackup +yum-builddep -y pg_probackup.spec +rpmbuild -bs pg_probackup.spec +rpmbuild -ba pg_probackup.spec #2>&1 | tee -ai /app/out/build.log diff --git a/packaging/test/scripts/suse_forks.sh b/packaging/test/scripts/suse_forks.sh new file mode 100755 index 000000000..b83f1ddd9 --- /dev/null +++ b/packaging/test/scripts/suse_forks.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -xe +set -o pipefail +exit 0 diff --git a/po/LINGUAS b/po/LINGUAS new file mode 100644 index 000000000..562ba4cf0 --- /dev/null +++ b/po/LINGUAS @@ -0,0 +1 @@ +ru diff --git a/po/ru.po b/po/ru.po new file mode 100644 index 000000000..1263675c2 --- /dev/null +++ b/po/ru.po @@ -0,0 +1,1880 @@ +# Russian message translation file for pg_probackup +# Copyright (C) 2022 PostgreSQL Global Development Group +# This file is distributed under the same license as the pg_probackup (PostgreSQL) package. +# Vyacheslav Makarov , 2022. +msgid "" +msgstr "" +"Project-Id-Version: pg_probackup (PostgreSQL)\n" +"Report-Msgid-Bugs-To: bugs@postgrespro.ru\n" +"POT-Creation-Date: 2022-04-08 11:33+0300\n" +"PO-Revision-Date: 2022-MO-DA HO:MI+ZONE\n" +"Last-Translator: Vyacheslav Makarov \n" +"Language-Team: Russian \n" +"Language: ru\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n" + +#: src/help.c:84 +#, c-format +msgid "" +"\n" +"%s - utility to manage backup/recovery of PostgreSQL database.\n" +msgstr "" +"\n" +"%s - утилита для управления резервным копированием/восстановлением базы данных PostgreSQL.\n" + +#: src/help.c:86 +#, c-format +msgid "" +"\n" +" %s help [COMMAND]\n" +msgstr "" + +#: src/help.c:88 +#, c-format +msgid "" +"\n" +" %s version\n" +msgstr "" + +#: src/help.c:90 +#, c-format +msgid "" +"\n" +" %s init -B backup-path\n" +msgstr "" + +#: src/help.c:92 +#, c-format +msgid "" +"\n" +" %s set-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:93 src/help.c:791 +#, c-format +msgid " [-D pgdata-path]\n" +msgstr "" + +#: src/help.c:94 src/help.c:130 src/help.c:218 +#, c-format +msgid " [--external-dirs=external-directories-paths]\n" +msgstr "" + +#: src/help.c:95 src/help.c:132 src/help.c:305 src/help.c:731 src/help.c:794 +#, c-format +msgid " [--log-level-console=log-level-console]\n" +msgstr "" + +#: src/help.c:96 src/help.c:133 src/help.c:306 src/help.c:732 src/help.c:795 +#, c-format +msgid " [--log-level-file=log-level-file]\n" +msgstr "" + +#: src/help.c:97 src/help.c:134 src/help.c:307 src/help.c:733 src/help.c:796 +#, c-format +msgid " [--log-filename=log-filename]\n" +msgstr "" + +#: src/help.c:98 src/help.c:135 src/help.c:308 src/help.c:734 src/help.c:797 +#, c-format +msgid " [--error-log-filename=error-log-filename]\n" +msgstr "" + +#: src/help.c:99 src/help.c:136 src/help.c:309 src/help.c:735 src/help.c:798 +#, c-format +msgid " [--log-directory=log-directory]\n" +msgstr "" + +#: src/help.c:100 src/help.c:137 src/help.c:310 src/help.c:736 src/help.c:799 +#, c-format +msgid " [--log-rotation-size=log-rotation-size]\n" +msgstr "" + +#: src/help.c:101 src/help.c:800 +#, c-format +msgid " [--log-rotation-age=log-rotation-age]\n" +msgstr "" + +#: src/help.c:102 src/help.c:140 src/help.c:203 src/help.c:313 src/help.c:674 +#: src/help.c:801 +#, c-format +msgid " [--retention-redundancy=retention-redundancy]\n" +msgstr "" + +#: src/help.c:103 src/help.c:141 src/help.c:204 src/help.c:314 src/help.c:675 +#: src/help.c:802 +#, c-format +msgid " [--retention-window=retention-window]\n" +msgstr "" + +#: src/help.c:104 src/help.c:142 src/help.c:205 src/help.c:315 src/help.c:676 +#: src/help.c:803 +#, c-format +msgid " [--wal-depth=wal-depth]\n" +msgstr "" + +#: src/help.c:105 src/help.c:144 src/help.c:235 src/help.c:317 src/help.c:804 +#: src/help.c:948 +#, c-format +msgid " [--compress-algorithm=compress-algorithm]\n" +msgstr "" + +#: src/help.c:106 src/help.c:145 src/help.c:236 src/help.c:318 src/help.c:805 +#: src/help.c:949 +#, c-format +msgid " [--compress-level=compress-level]\n" +msgstr "" + +#: src/help.c:107 src/help.c:232 src/help.c:806 src/help.c:945 +#, c-format +msgid " [--archive-timeout=timeout]\n" +msgstr "" + +#: src/help.c:108 src/help.c:147 src/help.c:259 src/help.c:320 src/help.c:807 +#: src/help.c:1045 +#, c-format +msgid " [-d dbname] [-h host] [-p port] [-U username]\n" +msgstr "" + +#: src/help.c:109 src/help.c:149 src/help.c:174 src/help.c:219 src/help.c:237 +#: src/help.c:247 src/help.c:261 src/help.c:322 src/help.c:449 src/help.c:808 +#: src/help.c:906 src/help.c:950 src/help.c:994 src/help.c:1047 +#, c-format +msgid " [--remote-proto] [--remote-host]\n" +msgstr "" + +#: src/help.c:110 src/help.c:150 src/help.c:175 src/help.c:220 src/help.c:238 +#: src/help.c:248 src/help.c:262 src/help.c:323 src/help.c:450 src/help.c:809 +#: src/help.c:907 src/help.c:951 src/help.c:995 src/help.c:1048 +#, c-format +msgid " [--remote-port] [--remote-path] [--remote-user]\n" +msgstr "" + +#: src/help.c:111 src/help.c:151 src/help.c:176 src/help.c:221 src/help.c:239 +#: src/help.c:249 src/help.c:263 src/help.c:324 src/help.c:451 src/help.c:1049 +#, c-format +msgid " [--ssh-options]\n" +msgstr "" + +#: src/help.c:112 +#, c-format +msgid " [--restore-command=cmdline] [--archive-host=destination]\n" +msgstr "" + +#: src/help.c:113 src/help.c:178 +#, c-format +msgid " [--archive-port=port] [--archive-user=username]\n" +msgstr "" + +#: src/help.c:114 src/help.c:119 src/help.c:123 src/help.c:153 src/help.c:179 +#: src/help.c:188 src/help.c:194 src/help.c:209 src/help.c:214 src/help.c:222 +#: src/help.c:226 src/help.c:240 src/help.c:250 src/help.c:264 +#, c-format +msgid " [--help]\n" +msgstr "" + +#: src/help.c:116 +#, c-format +msgid "" +"\n" +" %s set-backup -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:117 +#, c-format +msgid " -i backup-id [--ttl=interval] [--expire-time=timestamp]\n" +msgstr "" + +#: src/help.c:118 +#, c-format +msgid " [--note=text]\n" +msgstr "" + +#: src/help.c:121 +#, c-format +msgid "" +"\n" +" %s show-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:122 +#, c-format +msgid " [--format=format]\n" +msgstr "" + +#: src/help.c:125 +#, c-format +msgid "" +"\n" +" %s backup -B backup-path -b backup-mode --instance=instance_name\n" +msgstr "" + +#: src/help.c:126 src/help.c:299 +#, c-format +msgid " [-D pgdata-path] [-C]\n" +msgstr "" + +#: src/help.c:127 src/help.c:300 +#, c-format +msgid " [--stream [-S slot-name] [--temp-slot]]\n" +msgstr "" + +#: src/help.c:128 src/help.c:301 +#, c-format +msgid " [--backup-pg-log] [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:129 src/help.c:168 src/help.c:302 src/help.c:433 +#, c-format +msgid " [--no-validate] [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:131 src/help.c:304 +#, c-format +msgid " [--no-sync]\n" +msgstr "" + +#: src/help.c:138 src/help.c:311 +#, c-format +msgid " [--log-rotation-age=log-rotation-age] [--no-color]\n" +msgstr "" + +#: src/help.c:139 src/help.c:312 +#, c-format +msgid " [--delete-expired] [--delete-wal] [--merge-expired]\n" +msgstr "" + +#: src/help.c:143 src/help.c:316 +#, c-format +msgid " [--compress]\n" +msgstr "" + +#: src/help.c:146 src/help.c:319 +#, c-format +msgid " [--archive-timeout=archive-timeout]\n" +msgstr "" + +#: src/help.c:148 src/help.c:260 src/help.c:321 src/help.c:1046 +#, c-format +msgid " [-w --no-password] [-W --password]\n" +msgstr "" + +#: src/help.c:152 +#, c-format +msgid " [--ttl=interval] [--expire-time=timestamp] [--note=text]\n" +msgstr "" + +#: src/help.c:156 +#, c-format +msgid "" +"\n" +" %s restore -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:157 src/help.c:431 +#, c-format +msgid " [-D pgdata-path] [-i backup-id] [-j num-threads]\n" +msgstr "" + +#: src/help.c:158 src/help.c:183 src/help.c:439 src/help.c:552 +#, c-format +msgid " [--recovery-target-time=time|--recovery-target-xid=xid\n" +msgstr "" + +#: src/help.c:159 src/help.c:184 src/help.c:440 src/help.c:553 +#, c-format +msgid " |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n" +msgstr "" + +#: src/help.c:160 src/help.c:185 src/help.c:441 src/help.c:554 +#, c-format +msgid " [--recovery-target-timeline=timeline]\n" +msgstr "" + +#: src/help.c:161 src/help.c:442 +#, c-format +msgid " [--recovery-target=immediate|latest]\n" +msgstr "" + +#: src/help.c:162 src/help.c:186 src/help.c:443 src/help.c:555 +#, c-format +msgid " [--recovery-target-name=target-name]\n" +msgstr "" + +#: src/help.c:163 src/help.c:444 +#, c-format +msgid " [--recovery-target-action=pause|promote|shutdown]\n" +msgstr "" + +#: src/help.c:164 src/help.c:445 src/help.c:793 +#, c-format +msgid " [--restore-command=cmdline]\n" +msgstr "" + +#: src/help.c:165 +#, c-format +msgid " [-R | --restore-as-replica] [--force]\n" +msgstr "" + +#: src/help.c:166 src/help.c:447 +#, c-format +msgid " [--primary-conninfo=primary_conninfo]\n" +msgstr "" + +#: src/help.c:167 src/help.c:448 +#, c-format +msgid " [-S | --primary-slot-name=slotname]\n" +msgstr "" + +#: src/help.c:169 +#, c-format +msgid " [-T OLDDIR=NEWDIR] [--progress]\n" +msgstr "" + +#: src/help.c:170 src/help.c:435 +#, c-format +msgid " [--external-mapping=OLDDIR=NEWDIR]\n" +msgstr "" + +#: src/help.c:171 +#, c-format +msgid " [--skip-external-dirs] [--no-sync]\n" +msgstr "" + +#: src/help.c:172 src/help.c:437 +#, c-format +msgid " [-I | --incremental-mode=none|checksum|lsn]\n" +msgstr "" + +#: src/help.c:173 +#, c-format +msgid " [--db-include | --db-exclude]\n" +msgstr "" + +#: src/help.c:177 +#, c-format +msgid " [--archive-host=hostname]\n" +msgstr "" + +#: src/help.c:181 +#, c-format +msgid "" +"\n" +" %s validate -B backup-path [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:182 src/help.c:551 +#, c-format +msgid " [-i backup-id] [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:187 +#, c-format +msgid " [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:190 +#, c-format +msgid "" +"\n" +" %s checkdb [-B backup-path] [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:191 +#, c-format +msgid " [-D pgdata-path] [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:192 src/help.c:603 +#, c-format +msgid " [--amcheck] [--skip-block-validation]\n" +msgstr "" + +#: src/help.c:193 +#, c-format +msgid " [--heapallindexed] [--checkunique]\n" +msgstr "" + +#: src/help.c:196 +#, c-format +msgid "" +"\n" +" %s show -B backup-path\n" +msgstr "" + +#: src/help.c:197 src/help.c:657 +#, c-format +msgid " [--instance=instance_name [-i backup-id]]\n" +msgstr "" + +#: src/help.c:198 +#, c-format +msgid " [--format=format] [--archive]\n" +msgstr "" + +#: src/help.c:199 +#, c-format +msgid " [--no-color] [--help]\n" +msgstr "" + +#: src/help.c:201 +#, c-format +msgid "" +"\n" +" %s delete -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:202 src/help.c:673 +#, c-format +msgid " [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:206 +#, c-format +msgid " [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n" +msgstr "" + +#: src/help.c:207 +#, c-format +msgid " [--delete-wal]\n" +msgstr "" + +#: src/help.c:208 +#, c-format +msgid " [--dry-run] [--no-validate] [--no-sync]\n" +msgstr "" + +#: src/help.c:211 +#, c-format +msgid "" +"\n" +" %s merge -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:212 +#, c-format +msgid " -i backup-id [--progress] [-j num-threads]\n" +msgstr "" + +#: src/help.c:213 src/help.c:730 +#, c-format +msgid " [--no-validate] [--no-sync]\n" +msgstr "" + +#: src/help.c:216 +#, c-format +msgid "" +"\n" +" %s add-instance -B backup-path -D pgdata-path\n" +msgstr "" + +#: src/help.c:217 src/help.c:225 src/help.c:904 +#, c-format +msgid " --instance=instance_name\n" +msgstr "" + +#: src/help.c:224 +#, c-format +msgid "" +"\n" +" %s del-instance -B backup-path\n" +msgstr "" + +#: src/help.c:228 +#, c-format +msgid "" +"\n" +" %s archive-push -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:229 src/help.c:244 src/help.c:942 src/help.c:990 +#, c-format +msgid " --wal-file-name=wal-file-name\n" +msgstr "" + +#: src/help.c:230 src/help.c:943 src/help.c:991 +#, c-format +msgid " [--wal-file-path=wal-file-path]\n" +msgstr "" + +#: src/help.c:231 src/help.c:245 src/help.c:944 src/help.c:992 +#, c-format +msgid " [-j num-threads] [--batch-size=batch_size]\n" +msgstr "" + +#: src/help.c:233 src/help.c:946 +#, c-format +msgid " [--no-ready-rename] [--no-sync]\n" +msgstr "" + +#: src/help.c:234 src/help.c:947 +#, c-format +msgid " [--overwrite] [--compress]\n" +msgstr "" + +#: src/help.c:242 +#, c-format +msgid "" +"\n" +" %s archive-get -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:243 +#, c-format +msgid " --wal-file-path=wal-file-path\n" +msgstr "" + +#: src/help.c:246 src/help.c:993 +#, c-format +msgid " [--no-validate-wal]\n" +msgstr "" + +#: src/help.c:252 +#, c-format +msgid "" +"\n" +" %s catchup -b catchup-mode\n" +msgstr "" + +#: src/help.c:253 src/help.c:1039 +#, c-format +msgid " --source-pgdata=path_to_pgdata_on_remote_server\n" +msgstr "" + +#: src/help.c:254 src/help.c:1040 +#, c-format +msgid " --destination-pgdata=path_to_local_dir\n" +msgstr "" + +#: src/help.c:255 +#, c-format +msgid " [--stream [-S slot-name] [--temp-slot | --perm-slot]]\n" +msgstr "" + +#: src/help.c:256 src/help.c:1042 +#, c-format +msgid " [-j num-threads]\n" +msgstr "" + +#: src/help.c:257 src/help.c:434 src/help.c:1043 +#, c-format +msgid " [-T OLDDIR=NEWDIR]\n" +msgstr "" + +#: src/help.c:258 src/help.c:1044 +#, c-format +msgid " [--exclude-path=path_prefix]\n" +msgstr "" + +#: src/help.c:270 +#, c-format +msgid "Read the website for details <%s>.\n" +msgstr "Подробнее читайте на сайте <%s>.\n" + +#: src/help.c:272 +#, c-format +msgid "Report bugs to <%s>.\n" +msgstr "Сообщайте об ошибках в <%s>.\n" + +#: src/help.c:279 +#, c-format +msgid "" +"\n" +"Unknown command. Try pg_probackup help\n" +"\n" +msgstr "" +"\n" +"Неизвестная команда. Попробуйте pg_probackup help\n" +"\n" + +#: src/help.c:285 +#, c-format +msgid "" +"\n" +"This command is intended for internal use\n" +"\n" +msgstr "" + +#: src/help.c:291 +#, c-format +msgid "" +"\n" +"%s init -B backup-path\n" +"\n" +msgstr "" + +#: src/help.c:292 +#, c-format +msgid "" +" -B, --backup-path=backup-path location of the backup storage area\n" +"\n" +msgstr "" + +#: src/help.c:298 +#, c-format +msgid "" +"\n" +"%s backup -B backup-path -b backup-mode --instance=instance_name\n" +msgstr "" + +#: src/help.c:303 src/help.c:792 +#, c-format +msgid " [-E external-directories-paths]\n" +msgstr "" + +#: src/help.c:325 +#, c-format +msgid "" +" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n" +"\n" +msgstr "" + +#: src/help.c:327 src/help.c:455 src/help.c:558 src/help.c:606 src/help.c:660 +#: src/help.c:679 src/help.c:739 src/help.c:812 src/help.c:895 src/help.c:910 +#: src/help.c:934 src/help.c:954 src/help.c:998 +#, c-format +msgid " -B, --backup-path=backup-path location of the backup storage area\n" +msgstr "" + +#: src/help.c:328 +#, c-format +msgid " -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n" +msgstr "" + +#: src/help.c:329 src/help.c:456 src/help.c:559 src/help.c:607 src/help.c:680 +#: src/help.c:740 src/help.c:813 src/help.c:896 +#, c-format +msgid " --instance=instance_name name of the instance\n" +msgstr "" + +#: src/help.c:330 src/help.c:458 src/help.c:608 src/help.c:814 src/help.c:911 +#, c-format +msgid " -D, --pgdata=pgdata-path location of the database storage area\n" +msgstr "" + +#: src/help.c:331 +#, c-format +msgid " -C, --smooth-checkpoint do smooth checkpoint before backup\n" +msgstr "" + +#: src/help.c:332 +#, c-format +msgid " --stream stream the transaction log and include it in the backup\n" +msgstr "" + +#: src/help.c:333 src/help.c:1054 +#, c-format +msgid " -S, --slot=SLOTNAME replication slot to use\n" +msgstr "" + +#: src/help.c:334 src/help.c:1055 +#, c-format +msgid " --temp-slot use temporary replication slot\n" +msgstr "" + +#: src/help.c:335 +#, c-format +msgid " --backup-pg-log backup of '%s' directory\n" +msgstr "" + +#: src/help.c:336 src/help.c:460 src/help.c:563 src/help.c:611 src/help.c:682 +#: src/help.c:743 src/help.c:960 src/help.c:1004 src/help.c:1058 +#, c-format +msgid " -j, --threads=NUM number of parallel threads\n" +msgstr "" + +#: src/help.c:337 src/help.c:462 src/help.c:562 src/help.c:610 src/help.c:683 +#: src/help.c:744 +#, c-format +msgid " --progress show progress\n" +msgstr "" + +#: src/help.c:338 +#, c-format +msgid " --no-validate disable validation after backup\n" +msgstr "" + +#: src/help.c:339 src/help.c:466 src/help.c:573 +#, c-format +msgid " --skip-block-validation set to validate only file-level checksum\n" +msgstr "" + +#: src/help.c:340 src/help.c:815 src/help.c:914 +#, c-format +msgid " -E --external-dirs=external-directories-paths\n" +msgstr "" + +#: src/help.c:341 src/help.c:816 src/help.c:915 +#, c-format +msgid " backup some directories not from pgdata \n" +msgstr "" + +#: src/help.c:342 src/help.c:817 src/help.c:916 +#, c-format +msgid " (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n" +msgstr "" + +#: src/help.c:343 +#, c-format +msgid " --no-sync do not sync backed up files to disk\n" +msgstr "" + +#: src/help.c:344 +#, c-format +msgid " --note=text add note to backup\n" +msgstr "" + +#: src/help.c:345 src/help.c:784 +#, c-format +msgid " (example: --note='backup before app update to v13.1')\n" +msgstr "" + +#: src/help.c:347 src/help.c:508 src/help.c:575 src/help.c:622 src/help.c:702 +#: src/help.c:748 src/help.c:820 +#, c-format +msgid "" +"\n" +" Logging options:\n" +msgstr "" + +#: src/help.c:348 src/help.c:509 src/help.c:576 src/help.c:623 src/help.c:703 +#: src/help.c:749 src/help.c:821 +#, c-format +msgid " --log-level-console=log-level-console\n" +msgstr "" + +#: src/help.c:349 src/help.c:510 src/help.c:577 src/help.c:624 src/help.c:704 +#: src/help.c:750 src/help.c:822 +#, c-format +msgid " level for console logging (default: info)\n" +msgstr "" + +#: src/help.c:350 src/help.c:353 src/help.c:511 src/help.c:514 src/help.c:578 +#: src/help.c:581 src/help.c:625 src/help.c:628 src/help.c:705 src/help.c:708 +#: src/help.c:751 src/help.c:754 src/help.c:823 src/help.c:826 +#, c-format +msgid " available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n" +msgstr "" + +#: src/help.c:351 src/help.c:512 src/help.c:579 src/help.c:626 src/help.c:706 +#: src/help.c:752 src/help.c:824 +#, c-format +msgid " --log-level-file=log-level-file\n" +msgstr "" + +#: src/help.c:352 src/help.c:513 src/help.c:580 src/help.c:627 src/help.c:707 +#: src/help.c:753 src/help.c:825 +#, c-format +msgid " level for file logging (default: off)\n" +msgstr "" + +#: src/help.c:354 src/help.c:515 src/help.c:582 src/help.c:629 src/help.c:709 +#: src/help.c:755 src/help.c:827 +#, c-format +msgid " --log-filename=log-filename\n" +msgstr "" + +#: src/help.c:355 src/help.c:516 src/help.c:583 src/help.c:630 src/help.c:710 +#: src/help.c:756 src/help.c:828 +#, c-format +msgid " filename for file logging (default: 'pg_probackup.log')\n" +msgstr "" + +#: src/help.c:356 src/help.c:517 src/help.c:584 src/help.c:711 src/help.c:757 +#: src/help.c:829 +#, c-format +msgid " support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n" +msgstr "" + +#: src/help.c:357 src/help.c:518 src/help.c:585 src/help.c:632 src/help.c:712 +#: src/help.c:758 src/help.c:830 +#, c-format +msgid " --error-log-filename=error-log-filename\n" +msgstr "" + +#: src/help.c:358 src/help.c:519 src/help.c:586 src/help.c:633 src/help.c:713 +#: src/help.c:759 src/help.c:831 +#, c-format +msgid " filename for error logging (default: none)\n" +msgstr "" + +#: src/help.c:359 src/help.c:520 src/help.c:587 src/help.c:634 src/help.c:714 +#: src/help.c:760 src/help.c:832 +#, c-format +msgid " --log-directory=log-directory\n" +msgstr "" + +#: src/help.c:360 src/help.c:521 src/help.c:588 src/help.c:635 src/help.c:715 +#: src/help.c:761 src/help.c:833 +#, c-format +msgid " directory for file logging (default: BACKUP_PATH/log)\n" +msgstr "" + +#: src/help.c:361 src/help.c:522 src/help.c:589 src/help.c:636 src/help.c:716 +#: src/help.c:762 src/help.c:834 +#, c-format +msgid " --log-rotation-size=log-rotation-size\n" +msgstr "" + +#: src/help.c:362 src/help.c:523 src/help.c:590 src/help.c:637 src/help.c:717 +#: src/help.c:763 src/help.c:835 +#, c-format +msgid " rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:363 src/help.c:524 src/help.c:591 src/help.c:638 src/help.c:718 +#: src/help.c:764 src/help.c:836 +#, c-format +msgid " available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n" +msgstr "" + +#: src/help.c:364 src/help.c:525 src/help.c:592 src/help.c:639 src/help.c:719 +#: src/help.c:765 src/help.c:837 +#, c-format +msgid " --log-rotation-age=log-rotation-age\n" +msgstr "" + +#: src/help.c:365 src/help.c:526 src/help.c:593 src/help.c:640 src/help.c:720 +#: src/help.c:766 src/help.c:838 +#, c-format +msgid " rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:366 src/help.c:527 src/help.c:594 src/help.c:641 src/help.c:721 +#: src/help.c:767 src/help.c:839 +#, c-format +msgid " available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n" +msgstr "" + +#: src/help.c:367 src/help.c:528 src/help.c:642 +#, c-format +msgid " --no-color disable the coloring of error and warning console messages\n" +msgstr "" + +#: src/help.c:369 src/help.c:687 src/help.c:841 +#, c-format +msgid "" +"\n" +" Retention options:\n" +msgstr "" + +#: src/help.c:370 src/help.c:688 +#, c-format +msgid " --delete-expired delete backups expired according to current\n" +msgstr "" + +#: src/help.c:371 src/help.c:373 +#, c-format +msgid " retention policy after successful backup completion\n" +msgstr "" + +#: src/help.c:372 src/help.c:690 +#, c-format +msgid " --merge-expired merge backups expired according to current\n" +msgstr "" + +#: src/help.c:374 src/help.c:692 +#, c-format +msgid " --delete-wal remove redundant files in WAL archive\n" +msgstr "" + +#: src/help.c:375 src/help.c:693 src/help.c:842 +#, c-format +msgid " --retention-redundancy=retention-redundancy\n" +msgstr "" + +#: src/help.c:376 src/help.c:694 src/help.c:843 +#, c-format +msgid " number of full backups to keep; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:377 src/help.c:695 src/help.c:844 +#, c-format +msgid " --retention-window=retention-window\n" +msgstr "" + +#: src/help.c:378 src/help.c:696 src/help.c:845 +#, c-format +msgid " number of days of recoverability; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:379 src/help.c:697 +#, c-format +msgid " --wal-depth=wal-depth number of latest valid backups per timeline that must\n" +msgstr "" + +#: src/help.c:380 src/help.c:698 +#, c-format +msgid " retain the ability to perform PITR; 0 disables; (default: 0)\n" +msgstr "" + +#: src/help.c:381 src/help.c:699 +#, c-format +msgid " --dry-run perform a trial run without any changes\n" +msgstr "" + +#: src/help.c:383 +#, c-format +msgid "" +"\n" +" Pinning options:\n" +msgstr "" + +#: src/help.c:384 src/help.c:778 +#, c-format +msgid " --ttl=interval pin backup for specified amount of time; 0 unpin\n" +msgstr "" + +#: src/help.c:385 src/help.c:779 +#, c-format +msgid " available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n" +msgstr "" + +#: src/help.c:386 src/help.c:780 +#, c-format +msgid " (example: --ttl=20d)\n" +msgstr "" + +#: src/help.c:387 src/help.c:781 +#, c-format +msgid " --expire-time=time pin backup until specified time stamp\n" +msgstr "" + +#: src/help.c:388 src/help.c:782 +#, c-format +msgid " (example: --expire-time='2024-01-01 00:00:00+03')\n" +msgstr "" + +#: src/help.c:390 src/help.c:849 src/help.c:967 +#, c-format +msgid "" +"\n" +" Compression options:\n" +msgstr "" + +#: src/help.c:391 src/help.c:850 src/help.c:968 +#, c-format +msgid " --compress alias for --compress-algorithm='zlib' and --compress-level=1\n" +msgstr "" + +#: src/help.c:392 src/help.c:851 src/help.c:969 +#, c-format +msgid " --compress-algorithm=compress-algorithm\n" +msgstr "" + +#: src/help.c:393 +#, c-format +msgid " available options: 'zlib', 'pglz', 'none' (default: none)\n" +msgstr "" + +#: src/help.c:394 src/help.c:853 src/help.c:971 +#, c-format +msgid " --compress-level=compress-level\n" +msgstr "" + +#: src/help.c:395 src/help.c:854 src/help.c:972 +#, c-format +msgid " level of compression [0-9] (default: 1)\n" +msgstr "" + +#: src/help.c:397 src/help.c:856 +#, c-format +msgid "" +"\n" +" Archive options:\n" +msgstr "" + +#: src/help.c:398 src/help.c:857 +#, c-format +msgid " --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n" +msgstr "" + +#: src/help.c:400 src/help.c:644 src/help.c:859 src/help.c:1066 +#, c-format +msgid "" +"\n" +" Connection options:\n" +msgstr "" + +#: src/help.c:401 src/help.c:645 src/help.c:860 src/help.c:1067 +#, c-format +msgid " -U, --pguser=USERNAME user name to connect as (default: current local user)\n" +msgstr "" + +#: src/help.c:402 src/help.c:646 src/help.c:861 src/help.c:1068 +#, c-format +msgid " -d, --pgdatabase=DBNAME database to connect (default: username)\n" +msgstr "" + +#: src/help.c:403 src/help.c:647 src/help.c:862 src/help.c:1069 +#, c-format +msgid " -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n" +msgstr "" + +#: src/help.c:404 src/help.c:648 src/help.c:863 src/help.c:1070 +#, c-format +msgid " -p, --pgport=PORT database server port (default: 5432)\n" +msgstr "" + +#: src/help.c:405 src/help.c:649 src/help.c:1071 +#, c-format +msgid " -w, --no-password never prompt for password\n" +msgstr "" + +#: src/help.c:406 +#, c-format +msgid " -W, --password force password prompt\n" +msgstr "" + +#: src/help.c:408 src/help.c:530 src/help.c:865 src/help.c:917 src/help.c:974 +#: src/help.c:1009 src/help.c:1074 +#, c-format +msgid "" +"\n" +" Remote options:\n" +msgstr "" + +#: src/help.c:409 src/help.c:531 src/help.c:866 src/help.c:918 src/help.c:975 +#: src/help.c:1010 src/help.c:1075 +#, c-format +msgid " --remote-proto=protocol remote protocol to use\n" +msgstr "" + +#: src/help.c:410 src/help.c:532 src/help.c:867 src/help.c:919 src/help.c:976 +#: src/help.c:1011 src/help.c:1076 +#, c-format +msgid " available options: 'ssh', 'none' (default: ssh)\n" +msgstr "" + +#: src/help.c:411 src/help.c:533 src/help.c:868 src/help.c:920 +#, c-format +msgid " --remote-host=destination remote host address or hostname\n" +msgstr "" + +#: src/help.c:412 src/help.c:534 src/help.c:869 src/help.c:921 src/help.c:978 +#: src/help.c:1013 src/help.c:1078 +#, c-format +msgid " --remote-port=port remote host port (default: 22)\n" +msgstr "" + +#: src/help.c:413 src/help.c:535 src/help.c:870 src/help.c:922 src/help.c:979 +#: src/help.c:1014 src/help.c:1079 +#, c-format +msgid " --remote-path=path path to directory with pg_probackup binary on remote host\n" +msgstr "" + +#: src/help.c:414 src/help.c:536 src/help.c:871 src/help.c:923 src/help.c:980 +#: src/help.c:1015 src/help.c:1080 +#, c-format +msgid " (default: current binary path)\n" +msgstr "" + +#: src/help.c:415 src/help.c:537 src/help.c:872 src/help.c:924 src/help.c:981 +#: src/help.c:1016 src/help.c:1081 +#, c-format +msgid " --remote-user=username user name for ssh connection (default: current user)\n" +msgstr "" + +#: src/help.c:416 src/help.c:538 src/help.c:873 src/help.c:925 src/help.c:982 +#: src/help.c:1017 src/help.c:1082 +#, c-format +msgid " --ssh-options=ssh_options additional ssh options (default: none)\n" +msgstr "" + +#: src/help.c:417 src/help.c:539 src/help.c:874 +#, c-format +msgid " (example: --ssh-options='-c cipher_spec -F configfile')\n" +msgstr "" + +#: src/help.c:419 src/help.c:881 +#, c-format +msgid "" +"\n" +" Replica options:\n" +msgstr "" + +#: src/help.c:420 src/help.c:882 +#, c-format +msgid " --master-user=user_name user name to connect to master (deprecated)\n" +msgstr "" + +#: src/help.c:421 src/help.c:883 +#, c-format +msgid " --master-db=db_name database to connect to master (deprecated)\n" +msgstr "" + +#: src/help.c:422 src/help.c:884 +#, c-format +msgid " --master-host=host_name database server host of master (deprecated)\n" +msgstr "" + +#: src/help.c:423 src/help.c:885 +#, c-format +msgid " --master-port=port database server port of master (deprecated)\n" +msgstr "" + +#: src/help.c:424 src/help.c:886 +#, c-format +msgid "" +" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n" +"\n" +msgstr "" + +#: src/help.c:430 +#, c-format +msgid "" +"\n" +"%s restore -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:432 +#, c-format +msgid " [--progress] [--force] [--no-sync]\n" +msgstr "" + +#: src/help.c:436 +#, c-format +msgid " [--skip-external-dirs]\n" +msgstr "" + +#: src/help.c:438 +#, c-format +msgid " [--db-include dbname | --db-exclude dbname]\n" +msgstr "" + +#: src/help.c:446 +#, c-format +msgid " [-R | --restore-as-replica]\n" +msgstr "" + +#: src/help.c:452 +#, c-format +msgid " [--archive-host=hostname] [--archive-port=port]\n" +msgstr "" + +#: src/help.c:453 +#, c-format +msgid "" +" [--archive-user=username]\n" +"\n" +msgstr "" + +#: src/help.c:459 +#, c-format +msgid " -i, --backup-id=backup-id backup to restore\n" +msgstr "" + +#: src/help.c:463 +#, c-format +msgid " --force ignore invalid status of the restored backup\n" +msgstr "" + +#: src/help.c:464 +#, c-format +msgid " --no-sync do not sync restored files to disk\n" +msgstr "" + +#: src/help.c:465 +#, c-format +msgid " --no-validate disable backup validation during restore\n" +msgstr "" + +#: src/help.c:468 src/help.c:1060 +#, c-format +msgid " -T, --tablespace-mapping=OLDDIR=NEWDIR\n" +msgstr "" + +#: src/help.c:469 src/help.c:1061 +#, c-format +msgid " relocate the tablespace from directory OLDDIR to NEWDIR\n" +msgstr "" + +#: src/help.c:470 +#, c-format +msgid " --external-mapping=OLDDIR=NEWDIR\n" +msgstr "" + +#: src/help.c:471 +#, c-format +msgid " relocate the external directory from OLDDIR to NEWDIR\n" +msgstr "" + +#: src/help.c:472 +#, c-format +msgid " --skip-external-dirs do not restore all external directories\n" +msgstr "" + +#: src/help.c:474 +#, c-format +msgid "" +"\n" +" Incremental restore options:\n" +msgstr "" + +#: src/help.c:475 +#, c-format +msgid " -I, --incremental-mode=none|checksum|lsn\n" +msgstr "" + +#: src/help.c:476 +#, c-format +msgid " reuse valid pages available in PGDATA if they have not changed\n" +msgstr "" + +#: src/help.c:477 +#, c-format +msgid " (default: none)\n" +msgstr "" + +#: src/help.c:479 +#, c-format +msgid "" +"\n" +" Partial restore options:\n" +msgstr "" + +#: src/help.c:480 +#, c-format +msgid " --db-include dbname restore only specified databases\n" +msgstr "" + +#: src/help.c:481 +#, c-format +msgid " --db-exclude dbname do not restore specified databases\n" +msgstr "" + +#: src/help.c:483 +#, c-format +msgid "" +"\n" +" Recovery options:\n" +msgstr "" + +#: src/help.c:484 src/help.c:564 +#, c-format +msgid " --recovery-target-time=time time stamp up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:485 src/help.c:565 +#, c-format +msgid " --recovery-target-xid=xid transaction ID up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:486 src/help.c:566 +#, c-format +msgid " --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n" +msgstr "" + +#: src/help.c:487 src/help.c:567 +#, c-format +msgid " --recovery-target-inclusive=boolean\n" +msgstr "" + +#: src/help.c:488 src/help.c:568 +#, c-format +msgid " whether we stop just after the recovery target\n" +msgstr "" + +#: src/help.c:489 src/help.c:569 +#, c-format +msgid " --recovery-target-timeline=timeline\n" +msgstr "" + +#: src/help.c:490 src/help.c:570 +#, c-format +msgid " recovering into a particular timeline\n" +msgstr "" + +#: src/help.c:491 +#, c-format +msgid " --recovery-target=immediate|latest\n" +msgstr "" + +#: src/help.c:492 +#, c-format +msgid " end recovery as soon as a consistent state is reached or as late as possible\n" +msgstr "" + +#: src/help.c:493 src/help.c:571 +#, c-format +msgid " --recovery-target-name=target-name\n" +msgstr "" + +#: src/help.c:494 src/help.c:572 +#, c-format +msgid " the named restore point to which recovery will proceed\n" +msgstr "" + +#: src/help.c:495 +#, c-format +msgid " --recovery-target-action=pause|promote|shutdown\n" +msgstr "" + +#: src/help.c:496 +#, c-format +msgid " action the server should take once the recovery target is reached\n" +msgstr "" + +#: src/help.c:497 +#, c-format +msgid " (default: pause)\n" +msgstr "" + +#: src/help.c:498 src/help.c:818 +#, c-format +msgid " --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n" +msgstr "" + +#: src/help.c:500 +#, c-format +msgid "" +"\n" +" Standby options:\n" +msgstr "" + +#: src/help.c:501 +#, c-format +msgid " -R, --restore-as-replica write a minimal recovery.conf in the output directory\n" +msgstr "" + +#: src/help.c:502 +#, c-format +msgid " to ease setting up a standby server\n" +msgstr "" + +#: src/help.c:503 +#, c-format +msgid " --primary-conninfo=primary_conninfo\n" +msgstr "" + +#: src/help.c:504 +#, c-format +msgid " connection string to be used for establishing connection\n" +msgstr "" + +#: src/help.c:505 +#, c-format +msgid " with the primary server\n" +msgstr "" + +#: src/help.c:506 +#, c-format +msgid " -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n" +msgstr "" + +#: src/help.c:541 src/help.c:876 +#, c-format +msgid "" +"\n" +" Remote WAL archive options:\n" +msgstr "" + +#: src/help.c:542 src/help.c:877 +#, c-format +msgid " --archive-host=destination address or hostname for ssh connection to archive host\n" +msgstr "" + +#: src/help.c:543 src/help.c:878 +#, c-format +msgid " --archive-port=port port for ssh connection to archive host (default: 22)\n" +msgstr "" + +#: src/help.c:544 +#, c-format +msgid "" +" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n" +"\n" +msgstr "" + +#: src/help.c:550 +#, c-format +msgid "" +"\n" +"%s validate -B backup-path [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:556 +#, c-format +msgid "" +" [--skip-block-validation]\n" +"\n" +msgstr "" + +#: src/help.c:560 +#, c-format +msgid " -i, --backup-id=backup-id backup to validate\n" +msgstr "" + +#: src/help.c:595 src/help.c:722 src/help.c:768 +#, c-format +msgid "" +" --no-color disable the coloring of error and warning console messages\n" +"\n" +msgstr "" + +#: src/help.c:601 +#, c-format +msgid "" +"\n" +"%s checkdb [-B backup-path] [--instance=instance_name]\n" +msgstr "" + +#: src/help.c:602 +#, c-format +msgid " [-D pgdata-path] [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:604 +#, c-format +msgid "" +" [--heapallindexed] [--checkunique]\n" +"\n" +msgstr "" + +#: src/help.c:612 +#, c-format +msgid " --skip-block-validation skip file-level checking\n" +msgstr "" + +#: src/help.c:613 src/help.c:618 src/help.c:620 +#, c-format +msgid " can be used only with '--amcheck' option\n" +msgstr "" + +#: src/help.c:614 +#, c-format +msgid " --amcheck in addition to file-level block checking\n" +msgstr "" + +#: src/help.c:615 +#, c-format +msgid " check btree indexes via function 'bt_index_check()'\n" +msgstr "" + +#: src/help.c:616 +#, c-format +msgid " using 'amcheck' or 'amcheck_next' extensions\n" +msgstr "" + +#: src/help.c:617 +#, c-format +msgid " --heapallindexed also check that heap is indexed\n" +msgstr "" + +#: src/help.c:619 +#, c-format +msgid " --checkunique also check unique constraints\n" +msgstr "" + +#: src/help.c:631 +#, c-format +msgid " support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n" +msgstr "" + +#: src/help.c:650 src/help.c:1072 +#, c-format +msgid "" +" -W, --password force password prompt\n" +"\n" +msgstr "" + +#: src/help.c:656 +#, c-format +msgid "" +"\n" +"%s show -B backup-path\n" +msgstr "" + +#: src/help.c:658 +#, c-format +msgid "" +" [--format=format] [--archive]\n" +"\n" +msgstr "" + +#: src/help.c:661 +#, c-format +msgid " --instance=instance_name show info about specific instance\n" +msgstr "" + +#: src/help.c:662 +#, c-format +msgid " -i, --backup-id=backup-id show info about specific backups\n" +msgstr "" + +#: src/help.c:663 +#, c-format +msgid " --archive show WAL archive information\n" +msgstr "" + +#: src/help.c:664 +#, c-format +msgid " --format=format show format=PLAIN|JSON\n" +msgstr "" + +#: src/help.c:665 +#, c-format +msgid "" +" --no-color disable the coloring for plain format\n" +"\n" +msgstr "" + +#: src/help.c:671 +#, c-format +msgid "" +"\n" +"%s delete -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:672 +#, c-format +msgid " [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n" +msgstr "" + +#: src/help.c:677 +#, c-format +msgid "" +" [--no-validate] [--no-sync]\n" +"\n" +msgstr "" + +#: src/help.c:681 +#, c-format +msgid " -i, --backup-id=backup-id backup to delete\n" +msgstr "" + +#: src/help.c:684 src/help.c:745 +#, c-format +msgid " --no-validate disable validation during retention merge\n" +msgstr "" + +#: src/help.c:685 src/help.c:746 +#, c-format +msgid " --no-sync do not sync merged files to disk\n" +msgstr "" + +#: src/help.c:689 src/help.c:691 +#, c-format +msgid " retention policy\n" +msgstr "" + +#: src/help.c:700 +#, c-format +msgid " --status=backup_status delete all backups with specified status\n" +msgstr "" + +#: src/help.c:728 +#, c-format +msgid "" +"\n" +"%s merge -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:729 +#, c-format +msgid " -i backup-id [-j num-threads] [--progress]\n" +msgstr "" + +#: src/help.c:737 +#, c-format +msgid "" +" [--log-rotation-age=log-rotation-age]\n" +"\n" +msgstr "" + +#: src/help.c:741 +#, c-format +msgid " -i, --backup-id=backup-id backup to merge\n" +msgstr "" + +#: src/help.c:774 +#, c-format +msgid "" +"\n" +"%s set-backup -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:775 +#, c-format +msgid " -i backup-id\n" +msgstr "" + +#: src/help.c:776 +#, c-format +msgid "" +" [--ttl=interval] [--expire-time=time] [--note=text]\n" +"\n" +msgstr "" + +#: src/help.c:783 +#, c-format +msgid " --note=text add note to backup; 'none' to remove note\n" +msgstr "" + +#: src/help.c:790 +#, c-format +msgid "" +"\n" +"%s set-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:810 src/help.c:908 src/help.c:952 src/help.c:996 +#, c-format +msgid "" +" [--ssh-options]\n" +"\n" +msgstr "" + +#: src/help.c:846 +#, c-format +msgid " --wal-depth=wal-depth number of latest valid backups with ability to perform\n" +msgstr "" + +#: src/help.c:847 +#, c-format +msgid " the point in time recovery; disables; (default: 0)\n" +msgstr "" + +#: src/help.c:852 src/help.c:970 +#, c-format +msgid " available options: 'zlib','pglz','none' (default: 'none')\n" +msgstr "" + +#: src/help.c:879 +#, c-format +msgid " --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n" +msgstr "" + +#: src/help.c:892 +#, c-format +msgid "" +"\n" +"%s show-config -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:893 +#, c-format +msgid "" +" [--format=format]\n" +"\n" +msgstr "" + +#: src/help.c:897 +#, c-format +msgid "" +" --format=format show format=PLAIN|JSON\n" +"\n" +msgstr "" + +#: src/help.c:903 +#, c-format +msgid "" +"\n" +"%s add-instance -B backup-path -D pgdata-path\n" +msgstr "" + +#: src/help.c:905 +#, c-format +msgid " [-E external-directory-path]\n" +msgstr "" + +#: src/help.c:912 +#, c-format +msgid " --instance=instance_name name of the new instance\n" +msgstr "" + +#: src/help.c:926 src/help.c:983 src/help.c:1018 src/help.c:1083 +#, c-format +msgid "" +" (example: --ssh-options='-c cipher_spec -F configfile')\n" +"\n" +msgstr "" + +#: src/help.c:932 +#, c-format +msgid "" +"\n" +"%s del-instance -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:935 +#, c-format +msgid "" +" --instance=instance_name name of the instance to delete\n" +"\n" +msgstr "" + +#: src/help.c:941 +#, c-format +msgid "" +"\n" +"%s archive-push -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:955 src/help.c:999 +#, c-format +msgid " --instance=instance_name name of the instance to delete\n" +msgstr "" + +#: src/help.c:956 src/help.c:1002 +#, c-format +msgid " --wal-file-name=wal-file-name\n" +msgstr "" + +#: src/help.c:957 +#, c-format +msgid " name of the file to copy into WAL archive\n" +msgstr "" + +#: src/help.c:958 src/help.c:1000 +#, c-format +msgid " --wal-file-path=wal-file-path\n" +msgstr "" + +#: src/help.c:959 +#, c-format +msgid " relative destination path of the WAL archive\n" +msgstr "" + +#: src/help.c:961 +#, c-format +msgid " --batch-size=NUM number of files to be copied\n" +msgstr "" + +#: src/help.c:962 +#, c-format +msgid " --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n" +msgstr "" + +#: src/help.c:963 +#, c-format +msgid " --no-ready-rename do not rename '.ready' files in 'archive_status' directory\n" +msgstr "" + +#: src/help.c:964 +#, c-format +msgid " --no-sync do not sync WAL file to disk\n" +msgstr "" + +#: src/help.c:965 +#, c-format +msgid " --overwrite overwrite archived WAL file\n" +msgstr "" + +#: src/help.c:977 src/help.c:1012 src/help.c:1077 +#, c-format +msgid " --remote-host=hostname remote host address or hostname\n" +msgstr "" + +#: src/help.c:989 +#, c-format +msgid "" +"\n" +"%s archive-get -B backup-path --instance=instance_name\n" +msgstr "" + +#: src/help.c:1001 +#, c-format +msgid " relative destination path name of the WAL file on the server\n" +msgstr "" + +#: src/help.c:1003 +#, c-format +msgid " name of the WAL file to retrieve from the archive\n" +msgstr "" + +#: src/help.c:1005 +#, c-format +msgid " --batch-size=NUM number of files to be prefetched\n" +msgstr "" + +#: src/help.c:1006 +#, c-format +msgid " --prefetch-dir=path location of the store area for prefetched WAL files\n" +msgstr "" + +#: src/help.c:1007 +#, c-format +msgid " --no-validate-wal skip validation of prefetched WAL file before using it\n" +msgstr "" + +#: src/help.c:1024 +#, c-format +msgid "" +"\n" +"%s help [command]\n" +msgstr "" + +#: src/help.c:1025 +#, c-format +msgid "" +"%s command --help\n" +"\n" +msgstr "" + +#: src/help.c:1031 +#, c-format +msgid "" +"\n" +"%s version\n" +msgstr "" + +#: src/help.c:1032 +#, c-format +msgid "" +"%s --version\n" +"\n" +msgstr "" + +#: src/help.c:1038 +#, c-format +msgid "" +"\n" +"%s catchup -b catchup-mode\n" +msgstr "" + +#: src/help.c:1041 +#, c-format +msgid " [--stream [-S slot-name]] [--temp-slot | --perm-slot]\n" +msgstr "" + +#: src/help.c:1050 +#, c-format +msgid "" +" [--help]\n" +"\n" +msgstr "" + +#: src/help.c:1052 +#, c-format +msgid " -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n" +msgstr "" + +#: src/help.c:1053 +#, c-format +msgid " --stream stream the transaction log (only supported mode)\n" +msgstr "" + +#: src/help.c:1056 +#, c-format +msgid " -P --perm-slot create permanent replication slot\n" +msgstr "" + +#: src/help.c:1062 +#, c-format +msgid " -x, --exclude-path=path_prefix files with path_prefix (relative to pgdata) will be\n" +msgstr "" + +#: src/help.c:1063 +#, c-format +msgid " excluded from catchup (can be used multiple times)\n" +msgstr "" + +#: src/help.c:1064 +#, c-format +msgid " Dangerous option! Use at your own risk!\n" +msgstr "" diff --git a/src/archive.c b/src/archive.c index 4780db063..7d753c8b3 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2018-2019, Postgres Professional + * Portions Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -13,14 +13,6 @@ #include "utils/thread.h" #include "instr_time.h" -static int push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout); -#ifdef HAVE_LIBZ -static int push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout); -#endif static void *push_files(void *arg); static void *get_files(void *arg); static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, @@ -91,8 +83,19 @@ typedef struct WALSegno { char name[MAXFNAMELEN]; volatile pg_atomic_flag lock; + volatile pg_atomic_uint32 done; + struct WALSegno* prev; } WALSegno; +static int push_file_internal_uncompressed(WALSegno *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + uint32 archive_timeout); +#ifdef HAVE_LIBZ +static int push_file_internal_gz(WALSegno *wal_file_name, const char *pg_xlog_dir, + const char *archive_dir, bool overwrite, bool no_sync, + int compress_level, uint32 archive_timeout); +#endif + static int push_file(WALSegno *xlogfile, const char *archive_status_dir, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout, @@ -113,15 +116,13 @@ static parray *setup_push_filelist(const char *archive_status_dir, * Where archlog_path is $BACKUP_PATH/wal/instance_name */ void -do_archive_push(InstanceConfig *instance, char *wal_file_path, +do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename) { uint64 i; - char current_dir[MAXPGPATH]; - char pg_xlog_dir[MAXPGPATH]; - char archive_status_dir[MAXPGPATH]; - uint64 system_id; + /* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */ + char archive_status_dir[MAXPGPATH] = ""; bool is_compress = false; /* arrays with meta info for multi threaded backup */ @@ -141,31 +142,8 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, parray *batch_files = NULL; int n_threads; - if (wal_file_name == NULL) - elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); - - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - - /* verify that archive-push --instance parameter is valid */ - system_id = get_system_identifier(current_dir); - - if (instance->pgdata == NULL) - elog(ERROR, "Cannot read pg_probackup.conf for this instance"); - - if (system_id != instance->system_identifier) - elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instance->name, instance->system_identifier, system_id); - - if (instance->compress_alg == PGLZ_COMPRESS) - elog(ERROR, "Cannot use pglz for WAL compression"); - - join_path_components(pg_xlog_dir, current_dir, XLOGDIR); - join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); - - /* Create 'archlog_path' directory. Do nothing if it already exists. */ - //fio_mkdir(instance->arclog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + if (!no_ready_rename || batch_size > 1) + join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); #ifdef HAVE_LIBZ if (instance->compress_alg == ZLIB_COMPRESS) @@ -204,12 +182,13 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, { int rc; WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); + bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0; - rc = push_file(xlogfile, archive_status_dir, - pg_xlog_dir, instance->arclog_path, + rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir, + pg_xlog_dir, instanceState->instance_wal_subdir_path, overwrite, no_sync, instance->archive_timeout, - no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false, + no_ready_rename || first_wal, is_compress && IsXLogFileName(xlogfile->name) ? true : false, instance->compress_level); if (rc == 0) @@ -231,9 +210,9 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, archive_push_arg *arg = &(threads_args[i]); arg->first_filename = wal_file_name; - arg->archive_dir = instance->arclog_path; + arg->archive_dir = instanceState->instance_wal_subdir_path; arg->pg_xlog_dir = pg_xlog_dir; - arg->archive_status_dir = archive_status_dir; + arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL; arg->overwrite = overwrite; arg->compress = is_compress; arg->no_sync = no_sync; @@ -276,7 +255,7 @@ do_archive_push(InstanceConfig *instance, char *wal_file_path, /* Note, that we are leaking memory here, * because pushing into archive is a very - * time-sensetive operation, so we skip freeing stuff. + * time-sensitive operation, so we skip freeing stuff. */ push_done: @@ -356,37 +335,38 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, int compress_level) { int rc; - char wal_file_dummy[MAXPGPATH]; - - join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); elog(LOG, "pushing file \"%s\"", xlogfile->name); /* If compression is not required, then just copy it as is */ if (!is_compress) - rc = push_file_internal_uncompressed(xlogfile->name, pg_xlog_dir, + rc = push_file_internal_uncompressed(xlogfile, pg_xlog_dir, archive_dir, overwrite, no_sync, archive_timeout); #ifdef HAVE_LIBZ else - rc = push_file_internal_gz(xlogfile->name, pg_xlog_dir, archive_dir, + rc = push_file_internal_gz(xlogfile, pg_xlog_dir, archive_dir, overwrite, no_sync, compress_level, archive_timeout); #endif + pg_atomic_write_u32(&xlogfile->done, 1); + /* take '--no-ready-rename' flag into account */ - if (!no_ready_rename) + if (!no_ready_rename && archive_status_dir != NULL) { + char wal_file_dummy[MAXPGPATH]; char wal_file_ready[MAXPGPATH]; char wal_file_done[MAXPGPATH]; + join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready"); snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done"); canonicalize_path(wal_file_ready); canonicalize_path(wal_file_done); /* It is ok to rename status file in archive_status directory */ - elog(VERBOSE, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); + elog(LOG, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); /* do not error out, if rename failed */ if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) @@ -406,13 +386,14 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, * has the same checksum */ int -push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_dir, +push_file_internal_uncompressed(WALSegno *wal_file, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, uint32 archive_timeout) { FILE *in = NULL; int out = -1; char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ + const char *wal_file_name = wal_file->name; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; /* partial handling */ @@ -421,6 +402,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d int partial_try_count = 0; int partial_file_size = 0; bool partial_is_stale = true; + /* remote agent error message */ + char *errmsg = NULL; /* from path */ join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); @@ -432,7 +415,10 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); + } /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -445,8 +431,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } /* Already existing destination temp file is not an error condition */ } else @@ -476,15 +465,21 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } else /* Successfully created partial file */ break; } else + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } /* first round */ @@ -515,8 +510,11 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (out < 0) { if (!partial_is_stale) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", to_fullpath_part, archive_timeout); + } /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); @@ -524,19 +522,22 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); if (out < 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } part_opened: - elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); + elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); /* Check if possible to skip copying */ if (fileExists(to_fullpath, FIO_BACKUP_HOST)) { pg_crc32 crc32_src; pg_crc32 crc32_dst; - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false); if (crc32_src == crc32_dst) { @@ -559,6 +560,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath); } @@ -566,6 +568,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d } /* copy content */ + errno = 0; for (;;) { size_t read_len = 0; @@ -575,13 +578,15 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (ferror(in)) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot read source file \"%s\": %s", from_fullpath, strerror(errno)); } - if (read_len > 0 && fio_write(out, buf, read_len) != read_len) + if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to destination temp file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -593,10 +598,33 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* close source file */ fclose(in); + /* Writing is asynchronous in case of push in remote mode, so check agent status */ + if (fio_check_error_fd(out, &errmsg)) + { + fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "Cannot write to the remote file \"%s\": %s", + to_fullpath_part, errmsg); + } + + if (wal_file->prev != NULL) + { + while (!pg_atomic_read_u32(&wal_file->prev->done)) + { + if (thread_interrupted || interrupted) + { + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "Terminated while waiting for prev file"); + } + usleep(250); + } + } + /* close temp file */ if (fio_close(out) != 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot close temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -605,11 +633,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (!no_sync) { if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_part, strerror(errno)); + } } - elog(VERBOSE, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); + elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -617,6 +648,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", to_fullpath_part, to_fullpath, strerror(errno)); } @@ -634,13 +666,14 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d * has the same checksum */ int -push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, +push_file_internal_gz(WALSegno *wal_file, const char *pg_xlog_dir, const char *archive_dir, bool overwrite, bool no_sync, int compress_level, uint32 archive_timeout) { FILE *in = NULL; gzFile out = NULL; char *buf = pgut_malloc(OUT_BUF_SIZE); + const char *wal_file_name = wal_file->name; char from_fullpath[MAXPGPATH]; char to_fullpath[MAXPGPATH]; char to_fullpath_gz[MAXPGPATH]; @@ -652,6 +685,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, int partial_try_count = 0; int partial_file_size = 0; bool partial_is_stale = true; + /* remote agent errormsg */ + char *errmsg = NULL; /* from path */ join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); @@ -668,8 +703,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Open source file for read */ in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open source WAL file \"%s\": %s", from_fullpath, strerror(errno)); + } /* disable stdio buffering for input file */ setvbuf(in, NULL, _IONBF, BUFSIZ); @@ -679,8 +717,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } /* Already existing destination temp file is not an error condition */ } else @@ -710,16 +751,22 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (errno != EEXIST) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } else /* Successfully created partial file */ break; } else + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } /* first round */ @@ -750,8 +797,11 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (out == NULL) { if (!partial_is_stale) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", to_fullpath_gz_part, archive_timeout); + } /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); @@ -759,12 +809,15 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); if (out == NULL) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } part_opened: - elog(VERBOSE, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); + elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); /* Check if possible to skip copying, */ if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) @@ -772,9 +825,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, pg_crc32 crc32_src; pg_crc32 crc32_dst; - /* TODO: what if one of them goes missing? */ - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true); + crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); + crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false); if (crc32_src == crc32_dst) { @@ -797,6 +849,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, * so we must unlink partial file and exit with error. */ fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath_gz); } @@ -804,6 +857,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, } /* copy content */ + /* TODO: move to separate function */ for (;;) { size_t read_len = 0; @@ -813,6 +867,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (ferror(in)) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot read from source file \"%s\": %s", from_fullpath, strerror(errno)); } @@ -820,6 +875,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, get_gz_error(out, errno)); } @@ -831,10 +887,33 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* close source file */ fclose(in); - /* close temp file */ + /* Writing is asynchronous in case of push in remote mode, so check agent status */ + if (fio_check_error_fd_gz(out, &errmsg)) + { + fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", + to_fullpath_gz_part, errmsg); + } + + if (wal_file->prev != NULL) + { + while (!pg_atomic_read_u32(&wal_file->prev->done)) + { + if (thread_interrupted || interrupted) + { + pg_atomic_write_u32(&wal_file->done, 1); + elog(ERROR, "Terminated while waiting for prev file"); + } + usleep(250); + } + } + + /* close temp file, TODO: make it synchronous */ if (fio_gzclose(out) != 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); } @@ -843,11 +922,14 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (!no_sync) { if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) + { + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); + } } - elog(VERBOSE, "Rename \"%s\" to \"%s\"", + elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_gz_part, to_fullpath_gz); //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); @@ -856,6 +938,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) { fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + pg_atomic_write_u32(&wal_file->done, 1); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); } @@ -909,6 +992,15 @@ get_gz_error(gzFile gzf, int errnum) // } //} +static int +walSegnoCompareName(const void *f1, const void *f2) +{ + WALSegno *w1 = *(WALSegno**)f1; + WALSegno *w2 = *(WALSegno**)f2; + + return strcmp(w1->name, w2->name); +} + /* Look for files with '.ready' suffix in archive_status directory * and pack such files into batch sized array. */ @@ -916,14 +1008,15 @@ parray * setup_push_filelist(const char *archive_status_dir, const char *first_file, int batch_size) { - int i; WALSegno *xlogfile = NULL; parray *status_files = NULL; parray *batch_files = parray_new(); + size_t i; /* guarantee that first filename is in batch list */ - xlogfile = palloc(sizeof(WALSegno)); + xlogfile = palloc0(sizeof(WALSegno)); pg_atomic_init_flag(&xlogfile->lock); + pg_atomic_init_u32(&xlogfile->done, 0); snprintf(xlogfile->name, MAXFNAMELEN, "%s", first_file); parray_append(batch_files, xlogfile); @@ -954,8 +1047,9 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, if (strcmp(filename, first_file) == 0) continue; - xlogfile = palloc(sizeof(WALSegno)); + xlogfile = palloc0(sizeof(WALSegno)); pg_atomic_init_flag(&xlogfile->lock); + pg_atomic_init_u32(&xlogfile->done, 0); snprintf(xlogfile->name, MAXFNAMELEN, "%s", filename); parray_append(batch_files, xlogfile); @@ -964,6 +1058,13 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, break; } + parray_qsort(batch_files, walSegnoCompareName); + for (i = 1; i < parray_num(batch_files); i++) + { + xlogfile = (WALSegno*) parray_get(batch_files, i); + xlogfile->prev = (WALSegno*) parray_get(batch_files, i-1); + } + /* cleanup */ parray_walk(status_files, pgFileFree); parray_free(status_files); @@ -987,7 +1088,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file, */ void -do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, +do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, char *wal_file_name, int batch_size, bool validate_wal) { @@ -1025,8 +1126,8 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, join_path_components(absolute_wal_file_path, current_dir, wal_file_path); /* full filepath to WAL file in archive directory. - * backup_path/wal/instance_name/000000010000000000000001 */ - join_path_components(backup_wal_file_path, instance->arclog_path, wal_file_name); + * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ + join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name); INSTR_TIME_SET_CURRENT(start_time); if (num_threads > batch_size) @@ -1077,7 +1178,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, * copy requested file directly from archive. */ if (!next_wal_segment_exists(tli, segno, prefetch_dir, instance->xlog_seg_size)) - n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, tli, segno, num_threads, false, batch_size, instance->xlog_seg_size); @@ -1116,7 +1217,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, // rmtree(prefetch_dir, false); /* prefetch files */ - n_fetched = run_wal_prefetch(prefetch_dir, instance->arclog_path, + n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, tli, segno, num_threads, true, batch_size, instance->xlog_seg_size); @@ -1158,7 +1259,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false)) { fail_count = 0; - elog(INFO, "pg_probackup archive-get copied WAL file %s", wal_file_name); + elog(LOG, "pg_probackup archive-get copied WAL file %s", wal_file_name); n_fetched++; break; } @@ -1266,6 +1367,7 @@ uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, arg->thread_num = i+1; arg->files = batch_files; + arg->n_fetched = 0; } /* Run threads */ @@ -1378,11 +1480,11 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg); + rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg); if (rc == FILE_MISSING) #endif /* ... failing that, use uncompressed */ - rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg); /* When not in prefetch mode, try to use partial file */ if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) @@ -1392,13 +1494,13 @@ get_wal_file(const char *filename, const char *from_fullpath, #ifdef HAVE_LIBZ /* '.gz.partial' goes first ... */ snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg); + rc = fio_send_file_gz(from_partial, out, &errmsg); if (rc == FILE_MISSING) #endif { /* ... failing that, use '.partial' */ snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg); + rc = fio_send_file(from_partial, out, false, NULL, &errmsg); } if (rc == SEND_OK) @@ -1514,7 +1616,7 @@ get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ int exit_code = 0; - elog(VERBOSE, "Attempting to %s WAL file '%s'", + elog(LOG, "Attempting to %s WAL file '%s'", is_decompress ? "open compressed" : "open", from_path); /* open source file for read */ diff --git a/src/backup.c b/src/backup.c index 18cad5bf3..78c3512e9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3,7 +3,7 @@ * backup.c: backup DB cluster, archived WAL * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -13,9 +13,11 @@ #if PG_VERSION_NUM < 110000 #include "catalog/catalog.h" #endif +#if PG_VERSION_NUM < 120000 +#include "access/transam.h" +#endif #include "catalog/pg_tablespace.h" #include "pgtar.h" -#include "receivelog.h" #include "streamutil.h" #include @@ -25,53 +27,19 @@ #include "utils/thread.h" #include "utils/file.h" -static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ -static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; -static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; - -/* - * How long we should wait for streaming end in seconds. - * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 - */ -static uint32 stream_stop_timeout = 0; -/* Time in which we started to wait for streaming end */ -static time_t stream_stop_begin = 0; - -const char *progname = "pg_probackup"; +//const char *progname = "pg_probackup"; /* list of files contained in backup */ -static parray *backup_files_list = NULL; +parray *backup_files_list = NULL; /* We need critical section for datapagemap_add() in case of using threads */ static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; -/* - * We need to wait end of WAL streaming before execute pg_stop_backup(). - */ -typedef struct -{ - const char *basedir; - PGconn *conn; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; - - XLogRecPtr startpos; - TimeLineID starttli; -} StreamThreadArg; - -static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1}; - +// TODO: move to PGnodeInfo bool exclusive_backup = false; /* Is pg_start_backup() was executed */ -static bool backup_in_progress = false; -/* Is pg_stop_backup() was sent */ -static bool pg_stop_backup_is_sent = false; +bool backup_in_progress = false; /* * Backup routines @@ -80,20 +48,12 @@ static void backup_cleanup(bool fatal, void *userdata); static void *backup_files(void *arg); -static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); +static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, + PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); -static void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn); static void pg_switch_wal(PGconn *conn); -static void pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); -static int checkpoint_timeout(PGconn *backup_conn); -static XLogRecPtr wait_wal_lsn(XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir); - -static void *StreamLog(void *arg); -static void IdentifySystem(StreamThreadArg *stream_thread_arg); +static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); static void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn); @@ -103,38 +63,43 @@ static parray *get_database_map(PGconn *pg_startbackup_conn); static bool pgpro_support(PGconn *conn); /* Check functions */ -static bool pg_checksum_enable(PGconn *conn); +static bool pg_is_checksum_enabled(PGconn *conn); static bool pg_is_in_recovery(PGconn *conn); static bool pg_is_superuser(PGconn *conn); static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); static void confirm_block_size(PGconn *conn, const char *name, int blcksz); -static void set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); +static bool remove_excluded_files_criterion(void *value, void *exclude_args); +static void backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments); +static void process_file(int i, pgFile *file, backup_files_arg *arguments); + +static StopBackupCallbackParams stop_callback_params; static void backup_stopbackup_callback(bool fatal, void *userdata) { - PGconn *pg_startbackup_conn = (PGconn *) userdata; + StopBackupCallbackParams *st = (StopBackupCallbackParams *) userdata; /* * If backup is in progress, notify stop of backup to PostgreSQL */ if (backup_in_progress) { - elog(WARNING, "backup in progress, stop backup"); - pg_stop_backup(NULL, pg_startbackup_conn, NULL); /* don't care about stop_lsn in case of error */ + elog(WARNING, "A backup is in progress, stopping it."); + /* don't care about stop_lsn in case of error */ + pg_stop_backup_send(st->conn, st->server_version, current.from_replica, exclusive_backup, NULL); } } /* * Take a backup of a single postgresql instance. - * Move files from 'pgdata' to a subdirectory in 'backup_path'. + * Move files from 'pgdata' to a subdirectory in backup catalog. */ static void -do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) +do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, + PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) { int i; - char database_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ - char dst_backup_path[MAXPGPATH]; char label[1024]; XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; @@ -157,7 +122,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool char pretty_time[20]; char pretty_bytes[20]; - elog(LOG, "Database backup start"); + pgFile *src_pg_control_file = NULL; + + elog(INFO, "Database backup start"); if(current.external_dir_str) { external_dirs = make_external_directory_list(current.external_dir_str, @@ -165,12 +132,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool check_external_for_tablespaces(external_dirs, backup_conn); } - /* Clear ptrack files for not PTRACK backups */ - if (current.backup_mode != BACKUP_MODE_DIFF_PTRACK && nodeInfo->is_ptrack_enable) - pg_ptrack_clear(backup_conn, nodeInfo->ptrack_version_num); - /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time); + time2iso(label, lengthof(label), current.start_time, false); strncat(label, " with pg_probackup", lengthof(label) - strlen(" with pg_probackup")); @@ -181,18 +144,9 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); #else - current.tli = get_current_timeline_from_control(false); + current.tli = get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); #endif - /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal) - /* - * Do not wait start_lsn for stream backup. - * Because WAL streaming will start after pg_start_backup() in stream - * mode. - */ - wait_wal_lsn(current.start_lsn, true, current.tli, false, true, ERROR, false); - /* * In incremental backup mode ensure that already-validated * backup on current timeline exists and get its filelist. @@ -202,18 +156,23 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool current.backup_mode == BACKUP_MODE_DIFF_DELTA) { /* get list of backups already taken */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); if (prev_backup == NULL) { /* try to setup multi-timeline backup chain */ - elog(WARNING, "Valid backup on current timeline %u is not found, " + elog(WARNING, "Valid full backup on current timeline %u is not found, " "trying to look up on previous timelines", current.tli); - /* TODO: use read_timeline_history */ - tli_list = catalog_get_timelines(&instance_config); + tli_list = get_history_streaming(&instance_config.conn_opt, current.tli, backup_list); + if (!tli_list) + { + elog(WARNING, "Failed to obtain current timeline history file via replication protocol"); + /* fallback to using archive */ + tli_list = catalog_get_timelines(instanceState, &instance_config); + } if (parray_num(tli_list) == 0) elog(WARNING, "Cannot find valid backup on previous timelines, " @@ -235,13 +194,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool if (prev_backup) { - if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) - elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " - "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, base36enc(prev_backup->start_time), prev_backup->program_version); + if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) + elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " + "pg_probackup do not guarantee to be forward compatible. " + "Please upgrade pg_probackup binary.", + PROGRAM_VERSION, backup_id_of(prev_backup), prev_backup->program_version); - elog(INFO, "Parent backup: %s", base36enc(prev_backup->start_time)); + elog(INFO, "Parent backup: %s", backup_id_of(prev_backup)); /* Files of previous backup needed by DELTA backup */ prev_backup_filelist = get_backup_filelist(prev_backup, true); @@ -261,9 +220,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool { XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); + // new ptrack (>=2.0) is more robust and checks Start LSN if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) { - elog(ERROR, "LSN from ptrack_control %X/%X differs from Start LSN of previous backup %X/%X.\n" + elog(ERROR, "LSN from ptrack_control %X/%X is greater than Start LSN of previous backup %X/%X.\n" "Create new full backup before an incremental one.", (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), (uint32) (prev_backup->start_lsn >> 32), @@ -281,58 +241,52 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool "It may indicate that we are trying to backup PostgreSQL instance from the past.", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), - base36enc(prev_backup->start_time)); + backup_id_of(prev_backup)); /* Update running backup meta with START LSN */ write_backup(¤t, true); - pgBackupGetPath(¤t, database_path, lengthof(database_path), - DATABASE_DIR); - pgBackupGetPath(¤t, external_prefix, lengthof(external_prefix), - EXTERNAL_DIR); + /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ + if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !current.stream) + { + /* Check that archive_dir can be reached */ + if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) + elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", + instanceState->instance_wal_subdir_path, strerror(errno)); + + /* + * Do not wait start_lsn for stream backup. + * Because WAL streaming will start after pg_start_backup() in stream + * mode. + */ + wait_wal_lsn(instanceState->instance_wal_subdir_path, current.start_lsn, true, current.tli, false, true, ERROR, false); + } /* start stream replication */ - if (stream_wal) + if (current.stream) { - /* How long we should wait for streaming end after pg_stop_backup */ - stream_stop_timeout = checkpoint_timeout(backup_conn); - stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + char stream_xlog_path[MAXPGPATH]; - join_path_components(dst_backup_path, database_path, PG_XLOG_DIR); - fio_mkdir(dst_backup_path, DIR_PERMISSION, FIO_BACKUP_HOST); + join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR); + fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); - stream_thread_arg.basedir = dst_backup_path; + start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, + current.start_lsn, current.tli, true); - /* - * Connect in replication mode to the server. + /* Make sure that WAL streaming is working + * PAGE backup in stream mode is waited twice, first for + * segment in WAL archive and then for streamed segment */ - stream_thread_arg.conn = pgut_connect_replication(instance_config.conn_opt.pghost, - instance_config.conn_opt.pgport, - instance_config.conn_opt.pgdatabase, - instance_config.conn_opt.pguser); - /* sanity */ - IdentifySystem(&stream_thread_arg); - - /* By default there are some error */ - stream_thread_arg.ret = 1; - /* we must use startpos as start_lsn from start_backup */ - stream_thread_arg.startpos = current.start_lsn; - stream_thread_arg.starttli = current.tli; - - thread_interrupted = false; - pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); + wait_wal_lsn(stream_xlog_path, current.start_lsn, true, current.tli, false, true, ERROR, true); } - /* initialize backup list */ + /* initialize backup's file list */ backup_files_list = parray_new(); + join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR); /* list files with the logical path. omit $PGDATA */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0); - else - dir_list_file(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + fio_list_dir(backup_files_list, instance_config.pgdata, + true, true, false, backup_logs, true, 0); /* * Get database_map (name to oid) for use in partial restore feature. @@ -370,23 +324,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool elog(ERROR, "PGDATA is almost empty. Either it was concurrently deleted or " "pg_probackup do not possess sufficient permissions to list PGDATA content"); - /* Calculate pgdata_bytes */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - if (file->external_dir_num != 0) - continue; - - if (S_ISDIR(file->mode)) - { - current.pgdata_bytes += 4096; - continue; - } - - current.pgdata_bytes += file->size; - } - + current.pgdata_bytes += calculate_datasize_of_filelist(backup_files_list); pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); elog(INFO, "PGDATA size: %s", pretty_bytes); @@ -406,14 +344,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* Extract information about files in backup_list parsing their names:*/ parse_filelist_filenames(backup_files_list, instance_config.pgdata); + elog(INFO, "Current Start LSN: %X/%X, TLI: %X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + current.tli); if (current.backup_mode != BACKUP_MODE_FULL) - { - elog(LOG, "Current tli: %X", current.tli); - elog(LOG, "Parent start_lsn: %X/%X", - (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn)); - elog(LOG, "start_lsn: %X/%X", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); - } + elog(INFO, "Parent Start LSN: %X/%X, TLI: %X", + (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), + prev_backup->tli); /* * Build page mapping in incremental mode. @@ -434,7 +371,8 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool * reading WAL segments present in archives up to the point * where this backup has started. */ - pagemap_isok = extractPageMap(arclog_path, instance_config.xlog_seg_size, + pagemap_isok = extractPageMap(instanceState->instance_wal_subdir_path, + instance_config.xlog_seg_size, prev_backup->start_lsn, prev_backup->tli, current.start_lsn, current.tli, tli_list); } @@ -443,15 +381,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* * Build the page map from ptrack information. */ - if (nodeInfo->ptrack_version_num >= 20) - make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - nodeInfo->ptrack_version_num, - prev_backup_start_lsn); - else if (nodeInfo->ptrack_version_num == 15 || - nodeInfo->ptrack_version_num == 16 || - nodeInfo->ptrack_version_num == 17) - make_pagemap_from_ptrack_1(backup_files_list, backup_conn); + make_pagemap_from_ptrack_2(backup_files_list, backup_conn, + nodeInfo->ptrack_schema, + nodeInfo->ptrack_version_num, + prev_backup_start_lsn); } time(&end_time); @@ -466,7 +399,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } /* - * Make directories before backup and setup threads at the same time + * Make directories before backup */ for (i = 0; i < parray_num(backup_files_list); i++) { @@ -485,16 +418,35 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool join_path_components(dirpath, temp, file->rel_path); } else - join_path_components(dirpath, database_path, file->rel_path); + join_path_components(dirpath, current.database_dir, file->rel_path); - elog(VERBOSE, "Create directory '%s'", dirpath); + elog(LOG, "Create directory '%s'", dirpath); fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); } - /* setup threads */ - pg_atomic_clear_flag(&file->lock); } + /* + * find pg_control file + * We'll copy it last + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(backup_files_list, &search_key, pgFileCompareRelPathWithExternal); + + if (control_file_elem_index < 0) + elog(ERROR, "File \"%s\" not found in PGDATA %s", XLOG_CONTROL_FILE, current.database_dir); + src_pg_control_file = (pgFile *)parray_get(backup_files_list, control_file_elem_index); + } + + /* setup thread locks */ + pfilearray_clear_locks(backup_files_list); + /* Sort by size for load balancing */ parray_qsort(backup_files_list, pgFileCompareSize); /* Sort the array for binary search */ @@ -519,14 +471,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool arg->nodeInfo = nodeInfo; arg->from_root = instance_config.pgdata; - arg->to_root = database_path; + arg->to_root = current.database_dir; arg->external_prefix = external_prefix; arg->external_dirs = external_dirs; arg->files_list = backup_files_list; arg->prev_filelist = prev_backup_filelist; arg->prev_start_lsn = prev_backup_start_lsn; - arg->conn_arg.conn = NULL; - arg->conn_arg.cancel_conn = NULL; arg->hdr_map = &(current.hdr_map); arg->thread_num = i+1; /* By default there are some error */ @@ -553,6 +503,26 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_isok = false; } + /* copy pg_control at very end */ + if (backup_isok) + { + + elog(progress ? INFO : LOG, "Progress: Backup file \"%s\"", + src_pg_control_file->rel_path); + + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + join_path_components(from_fullpath, instance_config.pgdata, src_pg_control_file->rel_path); + join_path_components(to_fullpath, current.database_dir, src_pg_control_file->rel_path); + + backup_non_data_file(src_pg_control_file, NULL, + from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, + true); + } + + + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -571,7 +541,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool } /* Notify end of backup */ - pg_stop_backup(¤t, backup_conn, nodeInfo); + pg_stop_backup(instanceState, ¤t, backup_conn, nodeInfo); /* In case of backup from replica >= 9.6 we must fix minRecPoint, * First we must find pg_control in backup_files_list. @@ -580,23 +550,14 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool { pgFile *pg_control = NULL; - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); + pg_control = src_pg_control_file; - if (tmp_file->external_dir_num == 0 && - (strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0)) - { - pg_control = tmp_file; - break; - } - } if (!pg_control) elog(ERROR, "Failed to find file \"%s\" in backup filelist.", XLOG_CONTROL_FILE); - set_min_recovery_point(pg_control, database_path, current.stop_lsn); + set_min_recovery_point(pg_control, current.database_dir, current.stop_lsn); } /* close and sync page header map */ @@ -611,47 +572,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* close ssh session in main thread */ fio_disconnect(); - /* Add archived xlog files into the list of files of this backup */ - if (stream_wal) - { - parray *xlog_files_list; - char pg_xlog_path[MAXPGPATH]; - char wal_full_path[MAXPGPATH]; - - /* Scan backup PG_XLOG_DIR */ - xlog_files_list = parray_new(); - join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR); - dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, true, 0, - FIO_BACKUP_HOST); - - /* TODO: Drop streamed WAL segments greater than stop_lsn */ - for (i = 0; i < parray_num(xlog_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - - join_path_components(wal_full_path, pg_xlog_path, file->rel_path); - - if (!S_ISREG(file->mode)) - continue; - - file->crc = pgFileGetCRC(wal_full_path, true, false); - file->write_size = file->size; - - /* overwrite rel_path, because now it is relative to - * /backup_dir/backups/instance_name/backup_id/database/pg_xlog/ - */ - pg_free(file->rel_path); - - file->rel_path = pgut_strdup(GetRelativePath(wal_full_path, database_path)); - file->name = last_dir_separator(file->rel_path); - - /* Now it is relative to /backup_dir/backups/instance_name/backup_id/database/ */ - } - - /* Add xlog files into the list of backed up files */ - parray_concat(backup_files_list, xlog_files_list); - parray_free(xlog_files_list); - } + /* + * Add archived xlog files into the list of files of this backup + * NOTHING TO DO HERE + */ /* write database map to file and add it to control file */ if (database_map) @@ -690,7 +614,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool /* construct fullpath */ if (file->external_dir_num == 0) - join_path_components(to_fullpath, database_path, file->rel_path); + join_path_components(to_fullpath, current.database_dir, file->rel_path); else { char external_dst[MAXPGPATH]; @@ -717,7 +641,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool "It may indicate that we are trying to backup PostgreSQL instance from the past.", (uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn), (uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn), - base36enc(prev_backup->stop_lsn)); + backup_id_of(prev_backup)); /* clean external directories list */ if (external_dirs) @@ -778,7 +702,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) /* Confirm that this server version is supported */ check_server_version(cur_conn, nodeInfo); - if (pg_checksum_enable(cur_conn)) + if (pg_is_checksum_enabled(cur_conn)) current.checksum_version = 1; else current.checksum_version = 0; @@ -786,7 +710,7 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) nodeInfo->checksum_version = current.checksum_version; if (current.checksum_version) - elog(LOG, "This PostgreSQL instance was initialized with data block checksums. " + elog(INFO, "This PostgreSQL instance was initialized with data block checksums. " "Data block corruption will be detected"); else elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " @@ -795,9 +719,9 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) if (nodeInfo->is_superuser) elog(WARNING, "Current PostgreSQL role is superuser. " - "It is not recommended to run backup or checkdb as superuser."); + "It is not recommended to run pg_probackup under superuser."); - StrNCpy(current.server_version, nodeInfo->server_version_str, + strlcpy(current.server_version, nodeInfo->server_version_str, sizeof(current.server_version)); return cur_conn; @@ -805,49 +729,101 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) /* * Entry point of pg_probackup BACKUP subcommand. + * + * if start_time == INVALID_BACKUP_ID then we can generate backup_id */ int -do_backup(time_t start_time, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs) +do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, + bool no_validate, bool no_sync, bool backup_logs, time_t start_time) { PGconn *backup_conn = NULL; PGNodeInfo nodeInfo; + time_t latest_backup_id = INVALID_BACKUP_ID; char pretty_bytes[20]; + if (!instance_config.pgdata) + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); + /* Initialize PGInfonode */ pgNodeInit(&nodeInfo); - if (!instance_config.pgdata) - elog(ERROR, "required parameter not specified: PGDATA " - "(-D, --pgdata)"); + /* Save list of external directories */ + if (instance_config.external_dir_str && + (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) + current.external_dir_str = instance_config.external_dir_str; + + /* Find latest backup_id */ + { + parray *backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); + + if (parray_num(backup_list) > 0) + latest_backup_id = ((pgBackup *)parray_get(backup_list, 0))->backup_id; + + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } + + /* Try to pick backup_id and create backup directory with BACKUP_CONTROL_FILE */ + if (start_time != INVALID_BACKUP_ID) + { + /* If user already choosed backup_id for us, then try to use it. */ + if (start_time <= latest_backup_id) + /* don't care about freeing base36enc_dup memory, we exit anyway */ + elog(ERROR, "Can't assign backup_id from requested start_time (%s), " + "this time must be later that backup %s", + base36enc(start_time), base36enc(latest_backup_id)); + + current.backup_id = start_time; + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + } + else + { + /* We can generate our own unique backup_id + * Sometimes (when we try to backup twice in one second) + * backup_id will be duplicated -> try more times. + */ + int attempts = 10; + + if (time(NULL) < latest_backup_id) + elog(ERROR, "Can't assign backup_id, there is already a backup in future (%s)", + base36enc(latest_backup_id)); + + do + { + current.backup_id = time(NULL); + pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); + if (current.backup_id == INVALID_BACKUP_ID) + sleep(1); + } + while (current.backup_id == INVALID_BACKUP_ID && attempts-- > 0); + } + + /* If creation of backup dir was unsuccessful, there will be WARNINGS in logs already */ + if (current.backup_id == INVALID_BACKUP_ID) + elog(ERROR, "Can't create backup directory"); /* Update backup status and other metainfo. */ current.status = BACKUP_STATUS_RUNNING; - current.start_time = start_time; + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + current.start_time = current.backup_id; - StrNCpy(current.program_version, PROGRAM_VERSION, + strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); current.compress_alg = instance_config.compress_alg; current.compress_level = instance_config.compress_level; - /* Save list of external directories */ - if (instance_config.external_dir_str && - (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) - current.external_dir_str = instance_config.external_dir_str; - elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instance_name, base36enc(start_time), pgBackupGetBackupMode(¤t), + PROGRAM_VERSION, instanceState->instance_name, backup_id_of(¤t), pgBackupGetBackupMode(¤t, false), current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", deparse_compress_alg(current.compress_alg), current.compress_level); - /* Create backup directory and BACKUP_CONTROL_FILE */ - if (pgBackupCreateDir(¤t)) - elog(ERROR, "Cannot create backup directory"); - if (!lock_backup(¤t, true)) + if (!lock_backup(¤t, true, true)) elog(ERROR, "Cannot lock backup %s directory", - base36enc(current.start_time)); + backup_id_of(¤t)); write_backup(¤t, true); /* set the error processing function for the backup process */ @@ -862,7 +838,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); if (current.from_replica) - elog(INFO, "Backup %s is going to be taken from standby", base36enc(start_time)); + elog(INFO, "Backup %s is going to be taken from standby", backup_id_of(¤t)); /* TODO, print PostgreSQL full version */ //elog(INFO, "PostgreSQL version: %s", nodeInfo.server_version_str); @@ -884,15 +860,16 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); if (nodeInfo.ptrack_version_num > 0) - nodeInfo.is_ptrack_enable = pg_ptrack_enable(backup_conn, nodeInfo.ptrack_version_num); + nodeInfo.is_ptrack_enabled = pg_is_ptrack_enabled(backup_conn, nodeInfo.ptrack_version_num); if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { + /* ptrack_version_num < 2.0 was already checked in get_ptrack_version() */ if (nodeInfo.ptrack_version_num == 0) elog(ERROR, "This PostgreSQL instance does not support ptrack"); else { - if (!nodeInfo.is_ptrack_enable) + if (!nodeInfo.is_ptrack_enabled) elog(ERROR, "Ptrack is disabled"); } } @@ -907,7 +884,7 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, add_note(¤t, set_backup_params->note); /* backup data */ - do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs); + do_backup_pg(instanceState, backup_conn, &nodeInfo, no_sync, backup_logs); pgut_atexit_pop(backup_cleanup, NULL); /* compute size of wal files of this backup stored in the archive */ @@ -949,20 +926,20 @@ do_backup(time_t start_time, pgSetBackupParams *set_backup_params, pretty_size(current.data_bytes + current.wal_bytes, pretty_bytes, lengthof(pretty_bytes)); else pretty_size(current.data_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Backup %s resident size: %s", base36enc(current.start_time), pretty_bytes); + elog(INFO, "Backup %s resident size: %s", backup_id_of(¤t), pretty_bytes); if (current.status == BACKUP_STATUS_OK || current.status == BACKUP_STATUS_DONE) - elog(INFO, "Backup %s completed", base36enc(current.start_time)); + elog(INFO, "Backup %s completed", backup_id_of(¤t)); else - elog(ERROR, "Backup %s failed", base36enc(current.start_time)); + elog(ERROR, "Backup %s failed", backup_id_of(¤t)); /* * After successful backup completion remove backups * which are expired according to retention policies */ if (delete_expired || merge_expired || delete_wal) - do_retention(); + do_retention(instanceState, no_validate, no_sync); return 0; } @@ -991,16 +968,16 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) if (nodeInfo->server_version < 90500) elog(ERROR, - "server version is %s, must be %s or higher", + "Server version is %s, must be %s or higher", nodeInfo->server_version_str, "9.5"); if (current.from_replica && nodeInfo->server_version < 90600) elog(ERROR, - "server version is %s, must be %s or higher for backup from replica", + "Server version is %s, must be %s or higher for backup from replica", nodeInfo->server_version_str, "9.6"); if (nodeInfo->pgpro_support) - res = pgut_execute(conn, "SELECT pgpro_edition()", 0, NULL); + res = pgut_execute(conn, "SELECT pg_catalog.pgpro_edition()", 0, NULL); /* * Check major version of connected PostgreSQL and major version of @@ -1008,10 +985,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) */ #ifdef PGPRO_VERSION if (!res) + { /* It seems we connected to PostgreSQL (not Postgres Pro) */ - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with PostgreSQL %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + if(strcmp(PGPRO_EDITION, "1C") != 0) + { + elog(ERROR, "%s was built with Postgres Pro %s %s, " + "but connection is made with PostgreSQL %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } + /* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C + * Check the major version + */ + if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) + elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s", + PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); + } else { if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && @@ -1050,12 +1038,12 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) * All system identifiers must be equal. */ void -check_system_identifiers(PGconn *conn, char *pgdata) +check_system_identifiers(PGconn *conn, const char *pgdata) { uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata); + system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ @@ -1094,7 +1082,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) res = pgut_execute(conn, "SELECT pg_catalog.current_setting($1)", 1, &name); if (PQntuples(res) != 1 || PQnfields(res) != 1) - elog(ERROR, "cannot get %s: %s", name, PQerrorMessage(conn)); + elog(ERROR, "Cannot get %s: %s", name, PQerrorMessage(conn)); block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); if ((endp && *endp) || block_size != blcksz) @@ -1108,7 +1096,7 @@ confirm_block_size(PGconn *conn, const char *name, int blcksz) /* * Notify start of backup to PostgreSQL server. */ -static void +void pg_start_backup(const char *label, bool smooth, pgBackup *backup, PGNodeInfo *nodeInfo, PGconn *conn) { @@ -1116,28 +1104,33 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, const char *params[2]; uint32 lsn_hi; uint32 lsn_lo; - params[0] = label; +#if PG_VERSION_NUM >= 150000 + elog(INFO, "wait for pg_backup_start()"); +#else + elog(INFO, "wait for pg_start_backup()"); +#endif + /* 2nd argument is 'fast'*/ params[1] = smooth ? "false" : "true"; - if (!exclusive_backup) - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2, false)", - 2, - params); - else - res = pgut_execute(conn, - "SELECT pg_catalog.pg_start_backup($1, $2)", - 2, - params); + res = pgut_execute(conn, +#if PG_VERSION_NUM >= 150000 + "SELECT pg_catalog.pg_backup_start($1, $2)", +#else + "SELECT pg_catalog.pg_start_backup($1, $2, false)", +#endif + 2, + params); /* * Set flag that pg_start_backup() was called. If an error will happen it * is necessary to call pg_stop_backup() in backup_cleanup(). */ backup_in_progress = true; - pgut_atexit_push(backup_stopbackup_callback, conn); + stop_callback_params.conn = conn; + stop_callback_params.server_version = nodeInfo->server_version; + pgut_atexit_push(backup_stopbackup_callback, &stop_callback_params); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -1146,7 +1139,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, PQclear(res); - if ((!stream_wal || current.backup_mode == BACKUP_MODE_DIFF_PAGE) && + if ((!backup->stream || backup->backup_mode == BACKUP_MODE_DIFF_PAGE) && !backup->from_replica && !(nodeInfo->server_version < 90600 && !nodeInfo->is_superuser)) @@ -1163,14 +1156,12 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup, * Switch to a new WAL segment. It should be called only for master. * For PG 9.5 it should be called only if pguser is superuser. */ -static void +void pg_switch_wal(PGconn *conn) { PGresult *res; - /* Remove annoying NOTICE messages generated by backend */ - res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); - PQclear(res); + pg_silent_client_messages(conn); #if PG_VERSION_NUM >= 100000 res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL); @@ -1190,7 +1181,7 @@ pgpro_support(PGconn *conn) PGresult *res; res = pgut_execute(conn, - "SELECT proname FROM pg_proc WHERE proname='pgpro_edition'", + "SELECT proname FROM pg_catalog.pg_proc WHERE proname='pgpro_edition'::name AND pronamespace='pg_catalog'::regnamespace::oid", 0, NULL); if (PQresultStatus(res) == PGRES_TUPLES_OK && @@ -1229,7 +1220,7 @@ get_database_map(PGconn *conn) */ res = pgut_execute_extended(conn, "SELECT oid, datname FROM pg_catalog.pg_database " - "WHERE datname NOT IN ('template1', 'template0')", + "WHERE datname NOT IN ('template1'::name, 'template0'::name)", 0, NULL, true, true); /* Don't error out, simply return NULL. See comment above. */ @@ -1249,7 +1240,7 @@ get_database_map(PGconn *conn) db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); /* get Oid */ - db_entry->dbOid = atoi(PQgetvalue(res, i, 0)); + db_entry->dbOid = atoll(PQgetvalue(res, i, 0)); /* get datname */ datname = PQgetvalue(res, i, 1); @@ -1267,7 +1258,7 @@ get_database_map(PGconn *conn) /* Check if ptrack is enabled in target instance */ static bool -pg_checksum_enable(PGconn *conn) +pg_is_checksum_enabled(PGconn *conn) { PGresult *res_db; @@ -1333,7 +1324,7 @@ pg_is_superuser(PGconn *conn) * previous segment. * * Flag 'in_stream_dir' determine whether we looking for WAL in 'pg_wal' directory or - * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' because, + * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' (current.stream) because, * for example, PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. * * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than @@ -1342,15 +1333,13 @@ pg_is_superuser(PGconn *conn) * Returns target LSN if such is found, failing that returns LSN of record prior to target LSN. * Returns InvalidXLogRecPtr if 'segment_only' flag is used. */ -static XLogRecPtr -wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, +XLogRecPtr +wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, bool in_prev_segment, bool segment_only, int timeout_elevel, bool in_stream_dir) { XLogSegNo targetSegNo; - char pg_wal_dir[MAXPGPATH]; char wal_segment_path[MAXPGPATH], - *wal_segment_dir, wal_segment[MAXFNAMELEN]; bool file_exists = false; uint32 try_count = 0, @@ -1368,6 +1357,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, GetXLogFileName(wal_segment, tli, targetSegNo, instance_config.xlog_seg_size); + join_path_components(wal_segment_path, wal_segment_dir, wal_segment); /* * In pg_start_backup we wait for 'target_lsn' in 'pg_wal' directory if it is * stream and non-page backup. Page backup needs archived WAL files, so we @@ -1375,18 +1365,6 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, * * In pg_stop_backup it depends only on stream_wal. */ - if (in_stream_dir) - { - pgBackupGetPath2(¤t, pg_wal_dir, lengthof(pg_wal_dir), - DATABASE_DIR, PG_XLOG_DIR); - join_path_components(wal_segment_path, pg_wal_dir, wal_segment); - wal_segment_dir = pg_wal_dir; - } - else - { - join_path_components(wal_segment_path, arclog_path, wal_segment); - wal_segment_dir = arclog_path; - } /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ if (instance_config.archive_timeout > 0) @@ -1463,7 +1441,7 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, XLogRecPtr res; res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, - in_prev_segment, instance_config.xlog_seg_size); + in_prev_segment, instance_config.xlog_seg_size); if (!XLogRecPtrIsInvalid(res)) { @@ -1476,8 +1454,8 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, } sleep(1); - if (interrupted) - elog(ERROR, "Interrupted during waiting for WAL archiving"); + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving"); try_count++; /* Inform user if WAL segment is absent in first attempt */ @@ -1492,8 +1470,8 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, wal_delivery_str, wal_segment_path); } - if (!stream_wal && is_start_lsn && try_count == 30) - elog(WARNING, "By default pg_probackup assume WAL delivery method to be ARCHIVE. " + if (!current.stream && is_start_lsn && try_count == 30) + elog(WARNING, "By default pg_probackup assumes that WAL delivery method to be ARCHIVE. " "If continuous archiving is not set up, use '--stream' option to make autonomous backup. " "Otherwise check that continuous archiving works correctly."); @@ -1501,9 +1479,10 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, { if (file_exists) elog(timeout_elevel, "WAL segment %s was %s, " - "but target LSN %X/%X could not be archived in %d seconds", + "but target LSN %X/%X could not be %s in %d seconds", wal_segment, wal_delivery_str, - (uint32) (target_lsn >> 32), (uint32) target_lsn, timeout); + (uint32) (target_lsn >> 32), (uint32) target_lsn, + wal_delivery_str, timeout); /* If WAL segment doesn't exist or we wait for previous segment */ else elog(timeout_elevel, @@ -1516,69 +1495,258 @@ wait_wal_lsn(XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, } /* - * Notify end of backup to PostgreSQL server. + * Check stop_lsn (returned from pg_stop_backup()) and update backup->stop_lsn */ -static void -pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, - PGNodeInfo *nodeInfo) +void +wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup) { - PGconn *conn; - PGresult *res; - PGresult *tablespace_map_content = NULL; - uint32 lsn_hi; - uint32 lsn_lo; - //XLogRecPtr restore_lsn = InvalidXLogRecPtr; - int pg_stop_backup_timeout = 0; - char path[MAXPGPATH]; - char backup_label[MAXPGPATH]; - FILE *fp; - pgFile *file; - size_t len; - char *val = NULL; - char *stop_backup_query = NULL; - bool stop_lsn_exists = false; - XLogRecPtr stop_backup_lsn_tmp = InvalidXLogRecPtr; + bool stop_lsn_exists = false; - /* - * We will use this values if there are no transactions between start_lsn - * and stop_lsn. + /* It is ok for replica to return invalid STOP LSN + * UPD: Apparently it is ok even for a master. */ - time_t recovery_time; - TransactionId recovery_xid; + if (!XRecOffIsValid(stop_lsn)) + { + XLogSegNo segno = 0; + XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - if (!backup_in_progress) - elog(ERROR, "backup is not in progress"); + /* + * Even though the value is invalid, it's expected postgres behaviour + * and we're trying to fix it below. + */ + elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* + * Note: even with gdb it is very hard to produce automated tests for + * contrecord + invalid LSN, so emulate it for manual testing. + */ + //lsn = lsn - XLOG_SEG_SIZE; + //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", + // (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + GetXLogSegNo(stop_lsn, segno, instance_config.xlog_seg_size); + + /* + * Note, that there is no guarantee that corresponding WAL file even exists. + * Replica may return LSN from future and keep staying in present. + * Or it can return invalid LSN. + * + * That's bad, since we want to get real LSN to save it in backup label file + * and to use it in WAL validation. + * + * So we try to do the following: + * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and + * look for the first valid record in it. + * It solves the problem of occasional invalid LSN on write-busy system. + * 2. Failing that, look for record in previous segment with endpoint + * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN + * on write-idle system. If that fails too, error out. + */ + + /* stop_lsn is pointing to a 0 byte of xlog segment */ + if (stop_lsn % instance_config.xlog_seg_size == 0) + { + /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ + wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, true, WARNING, backup->stream); + + /* Get the first record in segment with current stop_lsn */ + lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout); + + /* Check that returned LSN is valid and greater than stop_lsn */ + if (XLogRecPtrIsInvalid(lsn_tmp) || + !XRecOffIsValid(lsn_tmp) || + lsn_tmp < stop_lsn) + { + /* Backup from master should error out here */ + if (!backup->from_replica) + elog(ERROR, "Failed to get next WAL record after %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + + /* No luck, falling back to looking up for previous record */ + elog(WARNING, "Failed to get next WAL record after %X/%X, " + "looking for previous WAL record", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + + /* Despite looking for previous record there is not guarantee of success + * because previous record can be the contrecord. + */ + lsn_tmp = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + true, false, ERROR, backup->stream); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record prior to %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + } + } + /* stop lsn is aligned to xlog block size, just find next lsn */ + else if (stop_lsn % XLOG_BLCKSZ == 0) + { + /* Wait for segment with current stop_lsn */ + wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, true, ERROR, backup->stream); + + /* Get the next closest record in segment with current stop_lsn */ + lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, + instance_config.xlog_seg_size, + instance_config.archive_timeout, + stop_lsn); + + /* sanity */ + if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) + elog(ERROR, "Failed to get WAL record next to %X/%X", + (uint32) (stop_lsn >> 32), + (uint32) (stop_lsn)); + } + /* PostgreSQL returned something very illegal as STOP_LSN, error out */ + else + elog(ERROR, "Invalid stop_backup_lsn value %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* Setting stop_backup_lsn will set stop point for streaming */ + stop_backup_lsn = lsn_tmp; + stop_lsn_exists = true; + } + + elog(INFO, "stop_lsn: %X/%X", + (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); + + /* + * Wait for stop_lsn to be archived or streamed. + * If replica returned valid STOP_LSN of not actually existing record, + * look for previous record with endpoint >= STOP_LSN. + */ + if (!stop_lsn_exists) + stop_backup_lsn = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, + false, false, ERROR, backup->stream); - conn = pg_startbackup_conn; + backup->stop_lsn = stop_backup_lsn; +} - /* Remove annoying NOTICE messages generated by backend */ +/* Remove annoying NOTICE messages generated by backend */ +void +pg_silent_client_messages(PGconn *conn) +{ + PGresult *res; res = pgut_execute(conn, "SET client_min_messages = warning;", 0, NULL); PQclear(res); +} - /* Make proper timestamp format for parse_time() */ - res = pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); - PQclear(res); +void +pg_create_restore_point(PGconn *conn, time_t backup_start_time) +{ + PGresult *res; + const char *params[1]; + char name[1024]; - /* Create restore point - * Only if backup is from master. - * For PG 9.5 create restore point only if pguser is superuser. - */ - if (backup != NULL && !backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) - { - const char *params[1]; - char name[1024]; + snprintf(name, lengthof(name), "pg_probackup, backup_id %s", + base36enc(backup_start_time)); + params[0] = name; - snprintf(name, lengthof(name), "pg_probackup, backup_id %s", - base36enc(backup->start_time)); - params[0] = name; + res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", + 1, params); + PQclear(res); +} - res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", - 1, params); - PQclear(res); - } +void +pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text) +{ + static const char + stop_exlusive_backup_query[] = + /* + * Stop the non-exclusive backup. Besides stop_lsn it returns from + * pg_stop_backup(false) copy of the backup label and tablespace map + * so they can be written to disk by the caller. + * TODO, question: add NULLs as backup_label and tablespace_map? + */ + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_stop_backup() as lsn", + stop_backup_on_master_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false, false)", + stop_backup_on_master_before10_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_master_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " lsn," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)", + /* + * In case of backup from replica >= 9.6 we do not trust minRecPoint + * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. + */ + stop_backup_on_replica_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_wal_replay_lsn()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false, false)", + stop_backup_on_replica_before10_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_xlog_replay_location()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_stop_backup(false)", + stop_backup_on_replica_after15_query[] = + "SELECT" + " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," + " current_timestamp(0)::timestamptz," + " pg_catalog.pg_last_wal_replay_lsn()," + " labelfile," + " spcmapfile" + " FROM pg_catalog.pg_backup_stop(false)"; + + const char * const stop_backup_query = + is_exclusive ? + stop_exlusive_backup_query : + server_version >= 150000 ? + (is_started_on_replica ? + stop_backup_on_replica_after15_query : + stop_backup_on_master_after15_query + ) : + (server_version >= 100000 ? + (is_started_on_replica ? + stop_backup_on_replica_query : + stop_backup_on_master_query + ) : + (is_started_on_replica ? + stop_backup_on_replica_before10_query : + stop_backup_on_master_before10_query + ) + ); + bool sent = false; + + /* Make proper timestamp format for parse_time(recovery_time) */ + pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); + // TODO: check result /* * send pg_stop_backup asynchronously because we could came @@ -1586,436 +1754,308 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn, * postgres archive_command problem and in this case we will * wait for pg_stop_backup() forever. */ + sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); + if (!sent) +#if PG_VERSION_NUM >= 150000 + elog(ERROR, "Failed to send pg_backup_stop query"); +#else + elog(ERROR, "Failed to send pg_stop_backup query"); +#endif - if (!pg_stop_backup_is_sent) + /* After we have sent pg_stop_backup, we don't need this callback anymore */ + pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); + + if (query_text) + *query_text = pgut_strdup(stop_backup_query); +} + +/* + * pg_stop_backup_consume -- get 'pg_stop_backup' query results + * side effects: + * - allocates memory for tablespace_map and backup_label contents, so it must freed by caller (if its not null) + * parameters: + * - + */ +void +pg_stop_backup_consume(PGconn *conn, int server_version, + bool is_exclusive, uint32 timeout, const char *query_text, + PGStopBackupResult *result) +{ + PGresult *query_result; + uint32 pg_stop_backup_timeout = 0; + enum stop_backup_query_result_column_numbers { + recovery_xid_colno = 0, + recovery_time_colno, + lsn_colno, + backup_label_colno, + tablespace_map_colno + }; + + /* and now wait */ + while (1) { - bool sent = false; + if (!PQconsumeInput(conn)) + elog(ERROR, "pg_stop backup() failed: %s", + PQerrorMessage(conn)); - if (!exclusive_backup) + if (PQisBusy(conn)) { - /* - * Stop the non-exclusive backup. Besides stop_lsn it returns from - * pg_stop_backup(false) copy of the backup label and tablespace map - * so they can be written to disk by the caller. - * In case of backup from replica >= 9.6 we do not trust minRecPoint - * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. - */ + pg_stop_backup_timeout++; + sleep(1); - /* current is used here because of cleanup */ - if (current.from_replica) - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," -#if PG_VERSION_NUM >= 100000 - " pg_catalog.pg_last_wal_replay_lsn()," -#else - " pg_catalog.pg_last_xlog_replay_location()," -#endif - " labelfile," - " spcmapfile" -#if PG_VERSION_NUM >= 100000 - " FROM pg_catalog.pg_stop_backup(false, false)"; + if (interrupted) + { + pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 + elog(ERROR, "Interrupted during waiting for pg_backup_stop"); #else - " FROM pg_catalog.pg_stop_backup(false)"; + elog(ERROR, "Interrupted during waiting for pg_stop_backup"); #endif - else - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" -#if PG_VERSION_NUM >= 100000 - " FROM pg_catalog.pg_stop_backup(false, false)"; + } + + if (pg_stop_backup_timeout == 1) + elog(INFO, "wait for pg_stop_backup()"); + + /* + * If postgres haven't answered in archive_timeout seconds, + * send an interrupt. + */ + if (pg_stop_backup_timeout > timeout) + { + pgut_cancel(conn); +#if PG_VERSION_NUM >= 150000 + elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); #else - " FROM pg_catalog.pg_stop_backup(false)"; + elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); #endif - + } } else { - stop_backup_query = "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn"; + query_result = PQgetResult(conn); + break; } - - sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); - pg_stop_backup_is_sent = true; - if (!sent) - elog(ERROR, "Failed to send pg_stop_backup query"); } - /* After we have sent pg_stop_backup, we don't need this callback anymore */ - pgut_atexit_pop(backup_stopbackup_callback, pg_startbackup_conn); - - /* - * Wait for the result of pg_stop_backup(), but no longer than - * archive_timeout seconds - */ - if (pg_stop_backup_is_sent && !in_cleanup) + /* Check successfull execution of pg_stop_backup() */ + if (!query_result) +#if PG_VERSION_NUM >= 150000 + elog(ERROR, "pg_backup_stop() failed"); +#else + elog(ERROR, "pg_stop_backup() failed"); +#endif + else { - res = NULL; - - while (1) + switch (PQresultStatus(query_result)) { - if (!PQconsumeInput(conn)) - elog(ERROR, "pg_stop backup() failed: %s", - PQerrorMessage(conn)); - - if (PQisBusy(conn)) - { - pg_stop_backup_timeout++; - sleep(1); - - if (interrupted) - { - pgut_cancel(conn); - elog(ERROR, "interrupted during waiting for pg_stop_backup"); - } - - if (pg_stop_backup_timeout == 1) - elog(INFO, "wait for pg_stop_backup()"); - - /* - * If postgres haven't answered in archive_timeout seconds, - * send an interrupt. - */ - if (pg_stop_backup_timeout > instance_config.archive_timeout) - { - pgut_cancel(conn); - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", - instance_config.archive_timeout); - } - } - else - { - res = PQgetResult(conn); + /* + * We should expect only PGRES_TUPLES_OK since pg_stop_backup + * returns tuples. + */ + case PGRES_TUPLES_OK: break; - } + default: + elog(ERROR, "Query failed: %s query was: %s", + PQerrorMessage(conn), query_text); } + backup_in_progress = false; + elog(INFO, "pg_stop backup() successfully executed"); + } - /* Check successfull execution of pg_stop_backup() */ - if (!res) - elog(ERROR, "pg_stop backup() failed"); - else - { - switch (PQresultStatus(res)) - { - /* - * We should expect only PGRES_TUPLES_OK since pg_stop_backup - * returns tuples. - */ - case PGRES_TUPLES_OK: - break; - default: - elog(ERROR, "query failed: %s query was: %s", - PQerrorMessage(conn), stop_backup_query); - } - elog(INFO, "pg_stop backup() successfully executed"); - } + /* get results and fill result structure */ + /* get&check recovery_xid */ + if (sscanf(PQgetvalue(query_result, 0, recovery_xid_colno), XID_FMT, &result->snapshot_xid) != 1) + elog(ERROR, + "Result of txid_snapshot_xmax() is invalid: %s", + PQgetvalue(query_result, 0, recovery_xid_colno)); - backup_in_progress = false; + /* get&check recovery_time */ + if (!parse_time(PQgetvalue(query_result, 0, recovery_time_colno), &result->invocation_time, true)) + elog(ERROR, + "Result of current_timestamp is invalid: %s", + PQgetvalue(query_result, 0, recovery_time_colno)); + + /* get stop_backup_lsn */ + { + uint32 lsn_hi; + uint32 lsn_lo; // char *target_lsn = "2/F578A000"; // XLogDataFromLSN(target_lsn, &lsn_hi, &lsn_lo); /* Extract timeline and LSN from results of pg_stop_backup() */ - XLogDataFromLSN(PQgetvalue(res, 0, 2), &lsn_hi, &lsn_lo); + XLogDataFromLSN(PQgetvalue(query_result, 0, lsn_colno), &lsn_hi, &lsn_lo); /* Calculate LSN */ - stop_backup_lsn_tmp = ((uint64) lsn_hi) << 32 | lsn_lo; + result->lsn = ((uint64) lsn_hi) << 32 | lsn_lo; + } - /* It is ok for replica to return invalid STOP LSN - * UPD: Apparently it is ok even for a master. - */ - if (!XRecOffIsValid(stop_backup_lsn_tmp)) - { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; + /* get backup_label_content */ + result->backup_label_content = NULL; + // if (!PQgetisnull(query_result, 0, backup_label_colno)) + if (!is_exclusive) + { + result->backup_label_content_len = PQgetlength(query_result, 0, backup_label_colno); + if (result->backup_label_content_len > 0) + result->backup_label_content = pgut_strndup(PQgetvalue(query_result, 0, backup_label_colno), + result->backup_label_content_len); + } else { + result->backup_label_content_len = 0; + } - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* get tablespace_map_content */ + result->tablespace_map_content = NULL; + // if (!PQgetisnull(query_result, 0, tablespace_map_colno)) + if (!is_exclusive) + { + result->tablespace_map_content_len = PQgetlength(query_result, 0, tablespace_map_colno); + if (result->tablespace_map_content_len > 0) + result->tablespace_map_content = pgut_strndup(PQgetvalue(query_result, 0, tablespace_map_colno), + result->tablespace_map_content_len); + } else { + result->tablespace_map_content_len = 0; + } +} - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + invalid LSN, so emulate it for manual testing. - */ - //stop_backup_lsn_tmp = stop_backup_lsn_tmp - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); +/* + * helper routine used to write backup_label and tablespace_map in pg_stop_backup() + */ +void +pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, + const void *data, size_t len, parray *file_list) +{ + FILE *fp; + pgFile *file; + char full_filename[MAXPGPATH]; + + join_path_components(full_filename, path, filename); + fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); + if (fp == NULL) + elog(ERROR, "Can't open %s file \"%s\": %s", + error_msg_filename, full_filename, strerror(errno)); + + if (fio_fwrite(fp, data, len) != len || + fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "Can't write %s file \"%s\": %s", + error_msg_filename, full_filename, strerror(errno)); - if (stream_wal) - { - pgBackupGetPath2(backup, stream_xlog_path, - lengthof(stream_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = arclog_path; + /* + * It's vital to check if files_list is initialized, + * because we could get here because the backup was interrupted + */ + if (file_list) + { + file = pgFileNew(full_filename, filename, true, 0, + FIO_BACKUP_HOST); - GetXLogSegNo(stop_backup_lsn_tmp, segno, instance_config.xlog_seg_size); + if (S_ISREG(file->mode)) + { + file->crc = pgFileGetCRC(full_filename, true, false); - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return invalid LSN. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid LSN on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN - * on write-idle system. If that fails too, error out. - */ + file->write_size = file->size; + file->uncompressed_size = file->size; + } + parray_append(file_list, file); + } +} - /* stop_lsn is pointing to a 0 byte of xlog segment */ - if (stop_backup_lsn_tmp % instance_config.xlog_seg_size == 0) - { - /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, true, WARNING, stream_wal); - - /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout); - - /* Check that returned LSN is valid and greater than stop_lsn */ - if (XLogRecPtrIsInvalid(lsn_tmp) || - !XRecOffIsValid(lsn_tmp) || - lsn_tmp < stop_backup_lsn_tmp) - { - /* Backup from master should error out here */ - if (!backup->from_replica) - elog(ERROR, "Failed to get next WAL record after %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - - /* No luck, falling back to looking up for previous record */ - elog(WARNING, "Failed to get next WAL record after %X/%X, " - "looking for previous WAL record", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - - /* Despite looking for previous record there is not guarantee of success - * because previous record can be the contrecord. - */ - lsn_tmp = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - true, false, ERROR, stream_wal); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record prior to %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - } - } - /* stop lsn is aligned to xlog block size, just find next lsn */ - else if (stop_backup_lsn_tmp % XLOG_BLCKSZ == 0) - { - /* Wait for segment with current stop_lsn */ - wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, true, ERROR, stream_wal); +/* + * Notify end of backup to PostgreSQL server. + */ +static void +pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, + PGNodeInfo *nodeInfo) +{ + PGStopBackupResult stop_backup_result; + char *xlog_path, stream_xlog_path[MAXPGPATH]; + /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ + int timeout = (instance_config.archive_timeout > 0) ? + instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; + char *query_text = NULL; + + /* Remove it ? */ + if (!backup_in_progress) + elog(ERROR, "Backup is not in progress"); - /* Get the next closest record in segment with current stop_lsn */ - lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout, - stop_backup_lsn_tmp); + pg_silent_client_messages(pg_startbackup_conn); - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record next to %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), - (uint32) (stop_backup_lsn_tmp)); - } - /* PostgreSQL returned something very illegal as STOP_LSN, error out */ - else - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); - - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; - } + /* Create restore point + * Only if backup is from master. + * For PG 9.5 create restore point only if pguser is superuser. + */ + if (!backup->from_replica && + !(nodeInfo->server_version < 90600 && + !nodeInfo->is_superuser)) //TODO: check correctness + pg_create_restore_point(pg_startbackup_conn, backup->start_time); - elog(LOG, "stop_lsn: %X/%X", - (uint32) (stop_backup_lsn_tmp >> 32), (uint32) (stop_backup_lsn_tmp)); + /* Execute pg_stop_backup using PostgreSQL connection */ + pg_stop_backup_send(pg_startbackup_conn, nodeInfo->server_version, backup->from_replica, exclusive_backup, &query_text); - /* Write backup_label and tablespace_map */ - if (!exclusive_backup) - { - Assert(PQnfields(res) >= 4); - pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); - - /* Write backup_label */ - join_path_components(backup_label, path, PG_BACKUP_LABEL_FILE); - fp = fio_fopen(backup_label, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "can't open backup label file \"%s\": %s", - backup_label, strerror(errno)); - - len = strlen(PQgetvalue(res, 0, 3)); - if (fio_fwrite(fp, PQgetvalue(res, 0, 3), len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "can't write backup label file \"%s\": %s", - backup_label, strerror(errno)); + /* + * Wait for the result of pg_stop_backup(), but no longer than + * archive_timeout seconds + */ + pg_stop_backup_consume(pg_startbackup_conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); - /* - * It's vital to check if backup_files_list is initialized, - * because we could get here because the backup was interrupted - */ - if (backup_files_list) - { - file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0, - FIO_BACKUP_HOST); + if (backup->stream) + { + join_path_components(stream_xlog_path, backup->database_dir, PG_XLOG_DIR); + xlog_path = stream_xlog_path; + } + else + xlog_path = instanceState->instance_wal_subdir_path; - file->crc = pgFileGetCRC(backup_label, true, false); + wait_wal_and_calculate_stop_lsn(xlog_path, stop_backup_result.lsn, backup); - file->write_size = file->size; - file->uncompressed_size = file->size; - parray_append(backup_files_list, file); - } - } + /* Write backup_label and tablespace_map */ + if (!exclusive_backup) + { + Assert(stop_backup_result.backup_label_content != NULL); - if (sscanf(PQgetvalue(res, 0, 0), XID_FMT, &recovery_xid) != 1) - elog(ERROR, - "result of txid_snapshot_xmax() is invalid: %s", - PQgetvalue(res, 0, 0)); - if (!parse_time(PQgetvalue(res, 0, 1), &recovery_time, true)) - elog(ERROR, - "result of current_timestamp is invalid: %s", - PQgetvalue(res, 0, 1)); - - /* Get content for tablespace_map from stop_backup results - * in case of non-exclusive backup - */ - if (!exclusive_backup) - val = PQgetvalue(res, 0, 4); + /* Write backup_label */ + pg_stop_backup_write_file_helper(backup->database_dir, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, + backup_files_list); + free(stop_backup_result.backup_label_content); + stop_backup_result.backup_label_content = NULL; + stop_backup_result.backup_label_content_len = 0; /* Write tablespace_map */ - if (!exclusive_backup && val && strlen(val) > 0) + if (stop_backup_result.tablespace_map_content != NULL) { - char tablespace_map[MAXPGPATH]; - - join_path_components(tablespace_map, path, PG_TABLESPACE_MAP_FILE); - fp = fio_fopen(tablespace_map, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "can't open tablespace map file \"%s\": %s", - tablespace_map, strerror(errno)); - - len = strlen(val); - if (fio_fwrite(fp, val, len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "can't write tablespace map file \"%s\": %s", - tablespace_map, strerror(errno)); - - if (backup_files_list) - { - file = pgFileNew(tablespace_map, PG_TABLESPACE_MAP_FILE, true, 0, - FIO_BACKUP_HOST); - if (S_ISREG(file->mode)) - { - file->crc = pgFileGetCRC(tablespace_map, true, false); - file->write_size = file->size; - } - - parray_append(backup_files_list, file); - } + pg_stop_backup_write_file_helper(backup->database_dir, PG_TABLESPACE_MAP_FILE, "tablespace map", + stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, + backup_files_list); + free(stop_backup_result.tablespace_map_content); + stop_backup_result.tablespace_map_content = NULL; + stop_backup_result.tablespace_map_content_len = 0; } - - if (tablespace_map_content) - PQclear(tablespace_map_content); - PQclear(res); } - /* Fill in fields if that is the correct end of backup. */ - if (backup != NULL) + if (backup->stream) { - char *xlog_path, - stream_xlog_path[MAXPGPATH]; - - /* - * Wait for stop_lsn to be archived or streamed. - * If replica returned valid STOP_LSN of not actually existing record, - * look for previous record with endpoint >= STOP_LSN. - */ - if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(stop_backup_lsn_tmp, false, backup->tli, - false, false, ERROR, stream_wal); - - if (stream_wal) - { - /* Wait for the completion of stream */ - pthread_join(stream_thread, NULL); - if (stream_thread_arg.ret == 1) - elog(ERROR, "WAL streaming failed"); - - pgBackupGetPath2(backup, stream_xlog_path, - lengthof(stream_xlog_path), - DATABASE_DIR, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = arclog_path; - - backup->stop_lsn = stop_backup_lsn; - backup->recovery_xid = recovery_xid; - - elog(LOG, "Getting the Recovery Time from WAL"); - - /* iterate over WAL from stop_backup lsn to start_backup lsn */ - if (!read_recovery_info(xlog_path, backup->tli, - instance_config.xlog_seg_size, - backup->start_lsn, backup->stop_lsn, - &backup->recovery_time)) - { - elog(LOG, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); - backup->recovery_time = recovery_time; - } + /* This function will also add list of xlog files + * to the passed filelist */ + if(wait_WAL_streaming_end(backup_files_list)) + elog(ERROR, "WAL streaming failed"); } -} -/* - * Retrieve checkpoint_timeout GUC value in seconds. - */ -static int -checkpoint_timeout(PGconn *backup_conn) -{ - PGresult *res; - const char *val; - const char *hintmsg; - int val_int; + backup->recovery_xid = stop_backup_result.snapshot_xid; - res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); - val = PQgetvalue(res, 0, 0); + elog(INFO, "Getting the Recovery Time from WAL"); - if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + /* iterate over WAL from stop_backup lsn to start_backup lsn */ + if (!read_recovery_info(xlog_path, backup->tli, + instance_config.xlog_seg_size, + backup->start_lsn, backup->stop_lsn, + &backup->recovery_time)) { - PQclear(res); - if (hintmsg) - elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, - hintmsg); - else - elog(ERROR, "Invalid value of checkout_timeout %s", val); + elog(INFO, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); + backup->recovery_time = stop_backup_result.invocation_time; } - PQclear(res); - - return val_int; + /* Cleanup */ + pg_free(query_text); } /* @@ -2033,7 +2073,7 @@ backup_cleanup(bool fatal, void *userdata) if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) { elog(WARNING, "Backup %s is running, setting its status to ERROR", - base36enc(current.start_time)); + backup_id_of(¤t)); current.end_time = time(NULL); current.status = BACKUP_STATUS_ERROR; write_backup(¤t, true); @@ -2052,8 +2092,6 @@ static void * backup_files(void *arg) { int i; - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; static time_t prev_time; backup_files_arg *arguments = (backup_files_arg *) arg; @@ -2065,11 +2103,17 @@ backup_files(void *arg) for (i = 0; i < n_backup_files_list; i++) { pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - pgFile *prev_file = NULL; /* We have already copied all directories */ if (S_ISDIR(file->mode)) continue; + /* + * Don't copy the pg_control file now, we'll copy it last + */ + if(file->external_dir_num == 0 && pg_strcasecmp(file->rel_path, XLOG_CONTROL_FILE) == 0) + { + continue; + } if (arguments->thread_num == 1) { @@ -2085,106 +2129,179 @@ backup_files(void *arg) } } + if (file->skip_cfs_nested) + continue; + if (!pg_atomic_test_set_flag(&file->lock)) continue; /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during backup"); + elog(ERROR, "Interrupted during backup"); - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_backup_files_list, file->rel_path); - - /* Handle zero sized files */ - if (file->size == 0) - { - file->write_size = 0; - continue; - } + elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_backup_files_list, file->rel_path); - /* construct destination filepath */ - if (file->external_dir_num == 0) + if (file->is_cfs) { - join_path_components(from_fullpath, arguments->from_root, file->rel_path); - join_path_components(to_fullpath, arguments->to_root, file->rel_path); + backup_cfs_segment(i, file, arguments); } else { - char external_dst[MAXPGPATH]; - char *external_path = parray_get(arguments->external_dirs, - file->external_dir_num - 1); + process_file(i, file, arguments); + } + } - makeExternalDirPathByNum(external_dst, + /* ssh connection to longer needed */ + fio_disconnect(); + + /* Data files transferring is successful */ + arguments->ret = 0; + + return NULL; +} + +static void +process_file(int i, pgFile *file, backup_files_arg *arguments) +{ + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + pgFile *prev_file = NULL; + + elog(progress ? INFO : LOG, "Progress: (%d/%zu). Process file \"%s\"", + i + 1, parray_num(arguments->files_list), file->rel_path); + + /* Handle zero sized files */ + if (file->size == 0) + { + file->write_size = 0; + return; + } + + /* construct from_fullpath & to_fullpath */ + if (file->external_dir_num == 0) + { + join_path_components(from_fullpath, arguments->from_root, file->rel_path); + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + } + else + { + char external_dst[MAXPGPATH]; + char *external_path = parray_get(arguments->external_dirs, + file->external_dir_num - 1); + + makeExternalDirPathByNum(external_dst, arguments->external_prefix, file->external_dir_num); - join_path_components(to_fullpath, external_dst, file->rel_path); - join_path_components(from_fullpath, external_path, file->rel_path); - } - - /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); + join_path_components(to_fullpath, external_dst, file->rel_path); + join_path_components(from_fullpath, external_path, file->rel_path); + } - /* Check that file exist in previous backup */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - pgFile **prev_file_tmp = NULL; - prev_file_tmp = (pgFile **) parray_bsearch(arguments->prev_filelist, - file, pgFileCompareRelPathWithExternal); - if (prev_file_tmp) - { - /* File exists in previous backup */ - file->exists_in_prev = true; - prev_file = *prev_file_tmp; - } - } + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) + { + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); + return; + } - /* backup file */ - if (file->is_datafile && !file->is_cfs) - { - backup_data_file(&(arguments->conn_arg), file, from_fullpath, to_fullpath, - arguments->prev_start_lsn, - current.backup_mode, - instance_config.compress_alg, - instance_config.compress_level, - arguments->nodeInfo->checksum_version, - arguments->nodeInfo->ptrack_version_num, - arguments->nodeInfo->ptrack_schema, - arguments->hdr_map, false); - } - else + /* Check that file exist in previous backup */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + pgFile **prevFileTmp = NULL; + prevFileTmp = (pgFile **) parray_bsearch(arguments->prev_filelist, + file, pgFileCompareRelPathWithExternal); + if (prevFileTmp) { - backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, - current.backup_mode, current.parent_backup, true); + /* File exists in previous backup */ + file->exists_in_prev = true; + prev_file = *prevFileTmp; } + } - if (file->write_size == FILE_NOT_FOUND) - continue; + /* backup file */ + if (file->is_datafile && !file->is_cfs) + { + backup_data_file(file, from_fullpath, to_fullpath, + arguments->prev_start_lsn, + current.backup_mode, + instance_config.compress_alg, + instance_config.compress_level, + arguments->nodeInfo->checksum_version, + arguments->hdr_map, false); + } + else + { + backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, + current.backup_mode, current.parent_backup, true); + } - if (file->write_size == BYTES_INVALID) - { - elog(VERBOSE, "Skipping the unchanged file: \"%s\"", from_fullpath); - continue; - } + if (file->write_size == FILE_NOT_FOUND) + return; - elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes", - from_fullpath, file->write_size); + if (file->write_size == BYTES_INVALID) + { + elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath); + return; } - /* ssh connection to longer needed */ - fio_disconnect(); + elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", + from_fullpath, file->write_size); - /* Close connection */ - if (arguments->conn_arg.conn) - pgut_disconnect(arguments->conn_arg.conn); +} - /* Data files transferring is successful */ - arguments->ret = 0; +static void +backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { + pgFile *data_file = file; + pgFile *cfm_file = NULL; + pgFile *data_bck_file = NULL; + pgFile *cfm_bck_file = NULL; - return NULL; + while (data_file->cfs_chain) + { + data_file = data_file->cfs_chain; + if (data_file->forkName == cfm) + cfm_file = data_file; + if (data_file->forkName == cfs_bck) + data_bck_file = data_file; + if (data_file->forkName == cfm_bck) + cfm_bck_file = data_file; + } + data_file = file; + if (data_file->relOid >= FirstNormalObjectId && cfm_file == NULL) + { + elog(ERROR, "'CFS' file '%s' have to have '%s.cfm' companion file", + data_file->rel_path, data_file->name); + } + + elog(LOG, "backup CFS segment %s, data_file=%s, cfm_file=%s, data_bck_file=%s, cfm_bck_file=%s", + data_file->name, data_file->name, cfm_file->name, data_bck_file == NULL? "NULL": data_bck_file->name, cfm_bck_file == NULL? "NULL": cfm_bck_file->name); + + /* storing cfs segment. processing corner case [PBCKP-287] stage 1. + * - when we do have data_bck_file we should skip both data_bck_file and cfm_bck_file if exists. + * they are removed by cfs_recover() during postgres start. + */ + if (data_bck_file) + { + if (cfm_bck_file) + cfm_bck_file->write_size = FILE_NOT_FOUND; + data_bck_file->write_size = FILE_NOT_FOUND; + } + /* else we store cfm_bck_file. processing corner case [PBCKP-287] stage 2. + * - when we do have cfm_bck_file only we should store it. + * it will replace cfm_file after postgres start. + */ + else if (cfm_bck_file) + process_file(i, cfm_bck_file, arguments); + + /* storing cfs segment in order cfm_file -> datafile to guarantee their consistency */ + /* cfm_file could be NULL for system tables. But we don't clear is_cfs flag + * for compatibility with older pg_probackup. */ + if (cfm_file) + process_file(i, cfm_file, arguments); + process_file(i, data_file, arguments); + elog(LOG, "Backup CFS segment %s done", data_file->name); } /* @@ -2214,11 +2331,12 @@ parse_filelist_filenames(parray *files, const char *root) */ if (strcmp(file->name, "pg_compression") == 0) { + /* processing potential cfs tablespace */ Oid tblspcOid; Oid dbOid; char tmp_rel_path[MAXPGPATH]; /* - * Check that the file is located under + * Check that pg_compression is located under * TABLESPACE_VERSION_DIRECTORY */ sscanf_result = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s/%u", @@ -2227,8 +2345,10 @@ parse_filelist_filenames(parray *files, const char *root) /* Yes, it is */ if (sscanf_result == 2 && strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY, - strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) - set_cfs_datafiles(files, root, file->rel_path, i); + strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) { + /* rewind index to the beginning of cfs tablespace */ + rewind_and_mark_cfs_datafiles(files, root, file->rel_path, i); + } } } @@ -2243,7 +2363,7 @@ parse_filelist_filenames(parray *files, const char *root) */ int unlogged_file_num = i - 1; pgFile *unlogged_file = (pgFile *) parray_get(files, - unlogged_file_num); + unlogged_file_num); unlogged_file_reloid = file->relOid; @@ -2251,11 +2371,10 @@ parse_filelist_filenames(parray *files, const char *root) (unlogged_file_reloid != 0) && (unlogged_file->relOid == unlogged_file_reloid)) { - pgFileFree(unlogged_file); - parray_remove(files, unlogged_file_num); + /* flagged to remove from list on stage 2 */ + unlogged_file->remove_from_list = true; unlogged_file_num--; - i--; unlogged_file = (pgFile *) parray_get(files, unlogged_file_num); @@ -2265,6 +2384,22 @@ parse_filelist_filenames(parray *files, const char *root) i++; } + + /* stage 2. clean up from temporary tables */ + parray_remove_if(files, remove_excluded_files_criterion, NULL, pgFileFree); +} + +static bool +remove_excluded_files_criterion(void *value, void *exclude_args) { + pgFile *file = (pgFile*)value; + return file->remove_from_list; +} + +static uint32 +hash_rel_seg(pgFile* file) +{ + uint32 hash = hash_mix32_2(file->relOid, file->segno); + return hash_mix32_2(hash, 0xcf5); } /* If file is equal to pg_compression, then we consider this tablespace as @@ -2278,43 +2413,95 @@ parse_filelist_filenames(parray *files, const char *root) * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression + * + * @returns index of first tablespace entry, i.e tblspcOid/TABLESPACE_VERSION_DIRECTORY */ static void -set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) +rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) { int len; int p; + int j; pgFile *prev_file; + pgFile *tmp_file; char *cfs_tblspc_path; + uint32 h; + + /* hash table for cfm files */ +#define HASHN 128 + parray *hashtab[HASHN] = {NULL}; + parray *bucket; + for (p = 0; p < HASHN; p++) + hashtab[p] = parray_new(); + cfs_tblspc_path = strdup(relative); if(!cfs_tblspc_path) elog(ERROR, "Out of memory"); len = strlen("/pg_compression"); cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; - elog(VERBOSE, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); + elog(LOG, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); for (p = (int) i; p >= 0; p--) { prev_file = (pgFile *) parray_get(files, (size_t) p); - elog(VERBOSE, "Checking file in cfs tablespace %s", prev_file->rel_path); + elog(LOG, "Checking file in cfs tablespace %s", prev_file->rel_path); - if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL) + if (strstr(prev_file->rel_path, cfs_tblspc_path) == NULL) { - if (S_ISREG(prev_file->mode) && prev_file->is_datafile) + elog(LOG, "Breaking on %s", prev_file->rel_path); + break; + } + + if (!S_ISREG(prev_file->mode)) + continue; + + h = hash_rel_seg(prev_file); + bucket = hashtab[h % HASHN]; + + if (prev_file->forkName == cfm || prev_file->forkName == cfm_bck || + prev_file->forkName == cfs_bck) + { + prev_file->skip_cfs_nested = true; + parray_append(bucket, prev_file); + } + else if (prev_file->is_datafile && prev_file->forkName == none) + { + elog(LOG, "Processing 'cfs' file %s", prev_file->rel_path); + /* have to mark as is_cfs even for system-tables for compatibility + * with older pg_probackup */ + prev_file->is_cfs = true; + prev_file->cfs_chain = NULL; + for (j = 0; j < parray_num(bucket); j++) { - elog(VERBOSE, "Setting 'is_cfs' on file %s, name %s", - prev_file->rel_path, prev_file->name); - prev_file->is_cfs = true; + tmp_file = parray_get(bucket, j); + elog(LOG, "Linking 'cfs' file '%s' to '%s'", + tmp_file->rel_path, prev_file->rel_path); + if (tmp_file->relOid == prev_file->relOid && + tmp_file->segno == prev_file->segno) + { + tmp_file->cfs_chain = prev_file->cfs_chain; + prev_file->cfs_chain = tmp_file; + parray_remove(bucket, j); + j--; + } } } - else + } + + for (p = 0; p < HASHN; p++) + { + bucket = hashtab[p]; + for (j = 0; j < parray_num(bucket); j++) { - elog(VERBOSE, "Breaking on %s", prev_file->rel_path); - break; + tmp_file = parray_get(bucket, j); + elog(WARNING, "Orphaned cfs related file '%s'", tmp_file->rel_path); } + parray_free(bucket); + hashtab[p] = NULL; } +#undef HASHN free(cfs_tblspc_path); } @@ -2365,160 +2552,13 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) pthread_mutex_unlock(&backup_pagemap_mutex); } + if (segno > 0) + pg_free(f.rel_path); pg_free(rel_path); -} - -/* - * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is - * set by pg_stop_backup(). - */ -static bool -stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) -{ - static uint32 prevtimeline = 0; - static XLogRecPtr prevpos = InvalidXLogRecPtr; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during WAL streaming"); - - /* we assume that we get called once at the end of each segment */ - if (segment_finished) - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), - (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - - /* - * Note that we report the previous, not current, position here. After a - * timeline switch, xlogpos points to the beginning of the segment because - * that's where we always begin streaming. Reporting the end of previous - * timeline isn't totally accurate, because the next timeline can begin - * slightly before the end of the WAL that we received on the previous - * timeline, but it's close enough for reporting purposes. - */ - if (prevtimeline != 0 && prevtimeline != timeline) - elog(LOG, _("switched to timeline %u at %X/%X\n"), - timeline, (uint32) (prevpos >> 32), (uint32) prevpos); - - if (!XLogRecPtrIsInvalid(stop_backup_lsn)) - { - if (xlogpos >= stop_backup_lsn) - { - stop_stream_lsn = xlogpos; - return true; - } - /* pg_stop_backup() was executed, wait for the completion of stream */ - if (stream_stop_begin == 0) - { - elog(INFO, "Wait for LSN %X/%X to be streamed", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); - - stream_stop_begin = time(NULL); - } - - if (time(NULL) - stream_stop_begin > stream_stop_timeout) - elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, - stream_stop_timeout); - } - - prevtimeline = timeline; - prevpos = xlogpos; - - return false; } -/* - * Start the log streaming - */ -static void * -StreamLog(void *arg) -{ - StreamThreadArg *stream_arg = (StreamThreadArg *) arg; - - /* - * Always start streaming at the beginning of a segment - */ - stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; - - /* Initialize timeout */ - stream_stop_begin = 0; - -#if PG_VERSION_NUM >= 100000 - /* if slot name was not provided for temp slot, use default slot name */ - if (!replication_slot && temp_slot) - replication_slot = "pg_probackup_slot"; -#endif - - -#if PG_VERSION_NUM >= 110000 - /* Create temp repslot */ - if (temp_slot) - CreateReplicationSlot(stream_arg->conn, replication_slot, - NULL, temp_slot, true, true, false); -#endif - - /* - * Start the replication - */ - elog(LOG, "started streaming WAL at %X/%X (timeline %u)", - (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, - stream_arg->starttli); - -#if PG_VERSION_NUM >= 90600 - { - StreamCtl ctl; - - MemSet(&ctl, 0, sizeof(ctl)); - - ctl.startpos = stream_arg->startpos; - ctl.timeline = stream_arg->starttli; - ctl.sysidentifier = NULL; - -#if PG_VERSION_NUM >= 100000 - ctl.walmethod = CreateWalDirectoryMethod(stream_arg->basedir, 0, true); - ctl.replication_slot = replication_slot; - ctl.stop_socket = PGINVALID_SOCKET; -#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 - ctl.temp_slot = temp_slot; -#endif -#else - ctl.basedir = (char *) stream_arg->basedir; -#endif - - ctl.stream_stop = stop_streaming; - ctl.standby_message_timeout = standby_message_timeout; - ctl.partial_suffix = NULL; - ctl.synchronous = false; - ctl.mark_done = false; - - if(ReceiveXlogStream(stream_arg->conn, &ctl) == false) - elog(ERROR, "Problem in receivexlog"); - -#if PG_VERSION_NUM >= 100000 - if (!ctl.walmethod->finish()) - elog(ERROR, "Could not finish writing WAL files: %s", - strerror(errno)); -#endif - } -#else - if(ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, - NULL, (char *) stream_arg->basedir, stop_streaming, - standby_message_timeout, NULL, false, false) == false) - elog(ERROR, "Problem in receivexlog"); -#endif - - elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", - (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); - stream_arg->ret = 0; - - PQfinish(stream_arg->conn); - stream_arg->conn = NULL; - - return NULL; -} - -static void +void check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) { PGresult *res; @@ -2584,60 +2624,34 @@ check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) } /* - * Run IDENTIFY_SYSTEM through a given connection and - * check system identifier and timeline are matching + * Calculate pgdata_bytes + * accepts (parray *) of (pgFile *) */ -void -IdentifySystem(StreamThreadArg *stream_thread_arg) +int64 +calculate_datasize_of_filelist(parray *filelist) { - PGresult *res; + int64 bytes = 0; + int i; - uint64 stream_conn_sysidentifier = 0; - char *stream_conn_sysidentifier_str; - TimeLineID stream_conn_tli = 0; + /* parray_num don't check for NULL */ + if (filelist == NULL) + return 0; - if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) + for (i = 0; i < parray_num(filelist); i++) { - PQfinish(stream_thread_arg->conn); - /* - * Error message already written in CheckServerVersionForStreaming(). - * There's no hope of recovering from a version mismatch, so don't - * retry. - */ - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - /* - * Identify server, obtain server system identifier and timeline - */ - res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING,"Could not send replication command \"%s\": %s", - "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); - PQfinish(stream_thread_arg->conn); - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } + pgFile *file = (pgFile *) parray_get(filelist, i); - stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); - stream_conn_tli = atoi(PQgetvalue(res, 0, 1)); - - /* Additional sanity, primary for PG 9.5, - * where system id can be obtained only via "IDENTIFY SYSTEM" - */ - if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) - elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); - - if (stream_conn_sysidentifier != instance_config.system_identifier) - elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " - "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", - stream_conn_sysidentifier, instance_config.system_identifier); + if (file->external_dir_num != 0 || file->excluded) + continue; - if (stream_conn_tli != current.tli) - elog(ERROR, "Timeline identifier mismatch. " - "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", - stream_conn_tli, current.tli); + if (S_ISDIR(file->mode)) + { + // TODO is a dir always 4K? + bytes += 4096; + continue; + } - PQclear(res); + bytes += file->size; + } + return bytes; } diff --git a/src/catalog.c b/src/catalog.c index 388f31e43..b29090789 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -23,11 +23,34 @@ static pgBackup* get_closest_backup(timelineInfo *tlinfo); static pgBackup* get_oldest_backup(timelineInfo *tlinfo); static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; static pgBackup *readBackupControlFile(const char *path); +static int create_backup_dir(pgBackup *backup, const char *backup_instance_path); -static bool exit_hook_registered = false; -static parray *lock_files = NULL; +static bool backup_lock_exit_hook_registered = false; +static parray *locks = NULL; -static timelineInfo * +static int grab_excl_lock_file(const char *backup_dir, const char *backup_id, bool strict); +static int grab_shared_lock_file(pgBackup *backup); +static int wait_shared_owners(pgBackup *backup); + + +static void unlink_lock_atexit(bool fatal, void *userdata); +static void unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive); +static void release_excl_lock_file(const char *backup_dir); +static void release_shared_lock_file(const char *backup_dir); + +#define LOCK_OK 0 +#define LOCK_FAIL_TIMEOUT 1 +#define LOCK_FAIL_ENOSPC 2 +#define LOCK_FAIL_EROFS 3 + +typedef struct LockInfo +{ + char backup_id[10]; + char backup_dir[MAXPGPATH]; + bool exclusive; +} LockInfo; + +timelineInfo * timelineInfoNew(TimeLineID tli) { timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); @@ -53,35 +76,32 @@ timelineInfoFree(void *tliInfo) if (tli->backups) { - parray_walk(tli->backups, pgBackupFree); + /* backups themselves should freed separately */ +// parray_walk(tli->backups, pgBackupFree); parray_free(tli->backups); } pfree(tliInfo); } -/* Iterate over locked backups and delete locks files */ -static void -unlink_lock_atexit(void) +/* Iterate over locked backups and unlock them */ +void +unlink_lock_atexit(bool unused_fatal, void *unused_userdata) { - int i; + int i; - if (lock_files == NULL) + if (locks == NULL) return; - for (i = 0; i < parray_num(lock_files); i++) + for (i = 0; i < parray_num(locks); i++) { - char *lock_file = (char *) parray_get(lock_files, i); - int res; - - res = fio_unlink(lock_file, FIO_BACKUP_HOST); - if (res != 0 && errno != ENOENT) - elog(WARNING, "%s: %s", lock_file, strerror(errno)); + LockInfo *lock = (LockInfo *) parray_get(locks, i); + unlock_backup(lock->backup_dir, lock->backup_id, lock->exclusive); } - parray_walk(lock_files, pfree); - parray_free(lock_files); - lock_files = NULL; + parray_walk(locks, pg_free); + parray_free(locks); + locks = NULL; } /* @@ -106,7 +126,7 @@ read_backup(const char *root_dir) */ void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name, bool strict) + bool strict) { pgBackup *tmp; @@ -120,69 +140,186 @@ write_backup_status(pgBackup *backup, BackupStatus status, return; } + /* overwrite control file only if status has changed */ + if (tmp->status == status) + { + pgBackupFree(tmp); + return; + } + backup->status = status; tmp->status = backup->status; tmp->root_dir = pgut_strdup(backup->root_dir); + /* lock backup in exclusive mode */ + if (!lock_backup(tmp, strict, true)) + elog(ERROR, "Cannot lock backup %s directory", backup_id_of(backup)); + write_backup(tmp, strict); pgBackupFree(tmp); } /* - * Create exclusive lockfile in the backup's directory. + * Lock backup in either exclusive or shared mode. + * "strict" flag allows to ignore "out of space" errors and should be + * used only by DELETE command to free disk space on filled up + * filesystem. + * + * Only read only tasks (validate, restore) are allowed to take shared locks. + * Changing backup metadata must be done with exclusive lock. + * + * Only one process can hold exclusive lock at any time. + * Exlusive lock - PID of process, holding the lock - is placed in + * lock file: BACKUP_LOCK_FILE. + * + * Multiple proccess are allowed to take shared locks simultaneously. + * Shared locks - PIDs of proccesses, holding the lock - are placed in + * separate lock file: BACKUP_RO_LOCK_FILE. + * When taking shared lock, a brief exclusive lock is taken. + * + * -> exclusive -> grab exclusive lock file and wait until all shared lockers are gone, return + * -> shared -> grab exclusive lock file, grab shared lock file, release exclusive lock file, return + * + * TODO: lock-timeout as parameter */ bool -lock_backup(pgBackup *backup, bool strict) +lock_backup(pgBackup *backup, bool strict, bool exclusive) { - char lock_file[MAXPGPATH]; - int fd; - char buffer[MAXPGPATH * 2 + 256]; - int ntries; - int len; - int encoded_pid; - pid_t my_pid, - my_p_pid; + int rc; + char lock_file[MAXPGPATH]; + bool enospc_detected = false; + LockInfo *lock = NULL; - join_path_components(lock_file, backup->root_dir, BACKUP_CATALOG_PID); + join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); + + rc = grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict); + + if (rc == LOCK_FAIL_TIMEOUT) + return false; + else if (rc == LOCK_FAIL_ENOSPC) + { + /* + * If we failed to take exclusive lock due to ENOSPC, + * then in lax mode treat such condition as if lock was taken. + */ + + enospc_detected = true; + if (strict) + return false; + } + else if (rc == LOCK_FAIL_EROFS) + { + /* + * If we failed to take exclusive lock due to EROFS, + * then in shared mode treat such condition as if lock was taken. + */ + return !exclusive; + } /* - * If the PID in the lockfile is our own PID or our parent's or - * grandparent's PID, then the file must be stale (probably left over from - * a previous system boot cycle). We need to check this because of the - * likelihood that a reboot will assign exactly the same PID as we had in - * the previous reboot, or one that's only one or two counts larger and - * hence the lockfile's PID now refers to an ancestor shell process. We - * allow pg_ctl to pass down its parent shell PID (our grandparent PID) - * via the environment variable PG_GRANDPARENT_PID; this is so that - * launching the postmaster via pg_ctl can be just as reliable as - * launching it directly. There is no provision for detecting - * further-removed ancestor processes, but if the init script is written - * carefully then all but the immediate parent shell will be root-owned - * processes and so the kill test will fail with EPERM. Note that we - * cannot get a false negative this way, because an existing postmaster - * would surely never launch a competing postmaster or pg_ctl process - * directly. + * We have exclusive lock, now there are following scenarios: + * + * 1. If we are for exlusive lock, then we must open the shared lock file + * and check if any of the processes listed there are still alive. + * If some processes are alive and are not going away in lock_timeout, + * then return false. + * + * 2. If we are here for non-exlusive lock, then write the pid + * into shared lock file and release the exclusive lock. */ - my_pid = getpid(); -#ifndef WIN32 - my_p_pid = getppid(); -#else + + if (exclusive) + rc = wait_shared_owners(backup); + else + rc = grab_shared_lock_file(backup); + + if (rc != 0) + { + /* + * Failed to grab shared lock or (in case of exclusive mode) shared lock owners + * are not going away in time, release the exclusive lock file and return in shame. + */ + release_excl_lock_file(backup->root_dir); + return false; + } + + if (!exclusive) + { + /* Shared lock file is grabbed, now we can release exclusive lock file */ + release_excl_lock_file(backup->root_dir); + } + + if (exclusive && !strict && enospc_detected) + { + /* We are in lax exclusive mode and EONSPC was encountered: + * once again try to grab exclusive lock file, + * because there is a chance that release of shared lock file in wait_shared_owners may have + * freed some space on filesystem, thanks to unlinking of BACKUP_RO_LOCK_FILE. + * If somebody concurrently acquired exclusive lock file first, then we should give up. + */ + if (grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict) == LOCK_FAIL_TIMEOUT) + return false; + + return true; + } /* - * Windows hasn't got getppid(), but doesn't need it since it's not using - * real kill() either... + * Arrange the unlocking at proc_exit. */ - my_p_pid = 0; -#endif + if (!backup_lock_exit_hook_registered) + { + pgut_atexit_push(unlink_lock_atexit, NULL); + backup_lock_exit_hook_registered = true; + } + + /* save lock metadata for later unlocking */ + lock = pgut_malloc(sizeof(LockInfo)); + snprintf(lock->backup_id, 10, "%s", backup_id_of(backup)); + snprintf(lock->backup_dir, MAXPGPATH, "%s", backup->root_dir); + lock->exclusive = exclusive; + + /* Use parray for lock release */ + if (locks == NULL) + locks = parray_new(); + parray_append(locks, lock); + + return true; +} + +/* + * Lock backup in exclusive mode + * Result codes: + * LOCK_OK Success + * LOCK_FAIL_TIMEOUT Failed to acquire lock in lock_timeout time + * LOCK_FAIL_ENOSPC Failed to acquire lock due to ENOSPC + * LOCK_FAIL_EROFS Failed to acquire lock due to EROFS + */ +int +grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) +{ + char lock_file[MAXPGPATH]; + int fd = 0; + char buffer[256]; + int ntries = LOCK_TIMEOUT; + int empty_tries = LOCK_STALE_TIMEOUT; + int len; + int encoded_pid; + + join_path_components(lock_file, root_dir, BACKUP_LOCK_FILE); /* * We need a loop here because of race conditions. But don't loop forever * (for example, a non-writable $backup_instance_path directory might cause a failure - * that won't go away). 100 tries seems like plenty. + * that won't go away). */ - for (ntries = 0;; ntries++) + do { + FILE *fp_out = NULL; + + if (interrupted) + elog(ERROR, "Interrupted while locking backup %s", backup_id); + /* * Try to create the lock file --- O_EXCL makes this atomic. * @@ -193,10 +330,21 @@ lock_backup(pgBackup *backup, bool strict) if (fd >= 0) break; /* Success; exit the retry loop */ + /* read-only fs is a special case */ + if (errno == EROFS) + { + elog(WARNING, "Could not create lock file \"%s\": %s", + lock_file, strerror(errno)); + return LOCK_FAIL_EROFS; + } + /* * Couldn't create the pid file. Probably it already exists. + * If file already exists or we have some permission problem (???), + * then retry; */ - if ((errno != EEXIST && errno != EACCES) || ntries > 100) +// if ((errno != EEXIST && errno != EACCES)) + if (errno != EEXIST) elog(ERROR, "Could not create lock file \"%s\": %s", lock_file, strerror(errno)); @@ -204,66 +352,118 @@ lock_backup(pgBackup *backup, bool strict) * Read the file to get the old owner's PID. Note race condition * here: file might have been deleted since we tried to create it. */ - fd = fio_open(lock_file, O_RDONLY, FIO_BACKUP_HOST); - if (fd < 0) + + fp_out = fopen(lock_file, "r"); + if (fp_out == NULL) { if (errno == ENOENT) - continue; /* race condition; try again */ - elog(ERROR, "Could not open lock file \"%s\": %s", - lock_file, strerror(errno)); + continue; /* race condition; try again */ + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); } - if ((len = fio_read(fd, buffer, sizeof(buffer) - 1)) < 0) - elog(ERROR, "Could not read lock file \"%s\": %s", - lock_file, strerror(errno)); - fio_close(fd); + len = fread(buffer, 1, sizeof(buffer) - 1, fp_out); + if (ferror(fp_out)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_out); + + /* + * There are several possible reasons for lock file + * to be empty: + * - system crash + * - process crash + * - race between writer and reader + * + * Consider empty file to be stale after LOCK_STALE_TIMEOUT attempts. + * + * TODO: alternatively we can write into temp file (lock_file_%pid), + * rename it and then re-read lock file to make sure, + * that we are successfully acquired the lock. + */ if (len == 0) - elog(ERROR, "Lock file \"%s\" is empty", lock_file); + { + if (empty_tries == 0) + { + elog(WARNING, "Lock file \"%s\" is empty", lock_file); + goto grab_lock; + } + + if ((empty_tries % LOG_FREQ) == 0) + elog(WARNING, "Waiting %u seconds on empty exclusive lock for backup %s", + empty_tries, backup_id); + + sleep(1); + /* + * waiting on empty lock file should not affect + * the timer for concurrent lockers (ntries). + */ + empty_tries--; + ntries++; + continue; + } - buffer[len] = '\0'; encoded_pid = atoi(buffer); if (encoded_pid <= 0) - elog(ERROR, "Bogus data in lock file \"%s\": \"%s\"", + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); + goto grab_lock; + } /* * Check to see if the other process still exists - * - * Per discussion above, my_pid, my_p_pid can be - * ignored as false matches. - * * Normally kill() will fail with ESRCH if the given PID doesn't * exist. */ - if (encoded_pid != my_pid && encoded_pid != my_p_pid) + if (encoded_pid == my_pid) + return LOCK_OK; + + if (kill(encoded_pid, 0) == 0) { - if (kill(encoded_pid, 0) == 0) + /* complain every fifth interval */ + if ((ntries % LOG_FREQ) == 0) { - elog(WARNING, "Process %d is using backup %s and still is running", - encoded_pid, base36enc(backup->start_time)); - return false; + elog(WARNING, "Process %d is using backup %s, and is still running", + encoded_pid, backup_id); + + elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", + ntries, backup_id); } + + sleep(1); + + /* try again */ + continue; + } + else + { + if (errno == ESRCH) + elog(WARNING, "Process %d which used backup %s no longer exists", + encoded_pid, backup_id); else - { - if (errno == ESRCH) - elog(WARNING, "Process %d which used backup %s no longer exists", - encoded_pid, base36enc(backup->start_time)); - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); } +grab_lock: /* * Looks like nobody's home. Unlink the file and try again to create * it. Need a loop because of possible race condition against other * would-be creators. */ if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) + { + if (errno == ENOENT) + continue; /* race condition, again */ elog(ERROR, "Could not remove old lock file \"%s\": %s", lock_file, strerror(errno)); - } + } + + } while (ntries--); + + /* Failed to acquire exclusive lock in time */ + if (fd <= 0) + return LOCK_FAIL_TIMEOUT; /* * Successfully created the file, now fill it. @@ -273,67 +473,369 @@ lock_backup(pgBackup *backup, bool strict) errno = 0; if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) { - int save_errno = errno; + int save_errno = errno; fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); - /* if write didn't set errno, assume problem is no disk space */ - errno = save_errno ? save_errno : ENOSPC; /* In lax mode if we failed to grab lock because of 'out of space error', * then treat backup as locked. * Only delete command should be run in lax mode. */ - if (!strict && errno == ENOSPC) - return true; - - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + if (!strict && save_errno == ENOSPC) + return LOCK_FAIL_ENOSPC; + else + elog(ERROR, "Could not write lock file \"%s\": %s", + lock_file, strerror(save_errno)); } + if (fio_flush(fd) != 0) { - int save_errno = errno; + int save_errno = errno; fio_close(fd); fio_unlink(lock_file, FIO_BACKUP_HOST); - errno = save_errno; - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + + /* In lax mode if we failed to grab lock because of 'out of space error', + * then treat backup as locked. + * Only delete command should be run in lax mode. + */ + if (!strict && save_errno == ENOSPC) + return LOCK_FAIL_ENOSPC; + else + elog(ERROR, "Could not flush lock file \"%s\": %s", + lock_file, strerror(save_errno)); } + if (fio_close(fd) != 0) { - int save_errno = errno; + int save_errno = errno; fio_unlink(lock_file, FIO_BACKUP_HOST); - errno = save_errno; - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(errno)); + + if (!strict && errno == ENOSPC) + return LOCK_FAIL_ENOSPC; + else + elog(ERROR, "Could not close lock file \"%s\": %s", + lock_file, strerror(save_errno)); } - /* - * Arrange to unlink the lock file(s) at proc_exit. - */ - if (!exit_hook_registered) +// elog(LOG, "Acquired exclusive lock for backup %s after %ds", +// backup_id_of(backup), +// LOCK_TIMEOUT - ntries + LOCK_STALE_TIMEOUT - empty_tries); + + return LOCK_OK; +} + +/* Wait until all shared lock owners are gone + * 0 - successs + * 1 - fail + */ +int +wait_shared_owners(pgBackup *backup) +{ + FILE *fp = NULL; + char buffer[256]; + pid_t encoded_pid = 0; + int ntries = LOCK_TIMEOUT; + char lock_file[MAXPGPATH]; + + join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); + + fp = fopen(lock_file, "r"); + if (fp == NULL && errno != ENOENT) + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + + /* iterate over pids in lock file */ + while (fp && fgets(buffer, sizeof(buffer), fp)) + { + encoded_pid = atoi(buffer); + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); + continue; + } + + /* wait until shared lock owners go away */ + do + { + if (interrupted) + elog(ERROR, "Interrupted while locking backup %s", + backup_id_of(backup)); + + if (encoded_pid == my_pid) + break; + + /* check if lock owner is still alive */ + if (kill(encoded_pid, 0) == 0) + { + /* complain from time to time */ + if ((ntries % LOG_FREQ) == 0) + { + elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", + encoded_pid, backup_id_of(backup)); + + elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, + backup_id_of(backup)); + } + + sleep(1); + + /* try again */ + continue; + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + + /* locker is dead */ + break; + + } while (ntries--); + } + + if (fp && ferror(fp)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + + if (fp) + fclose(fp); + + /* some shared owners are still alive */ + if (ntries <= 0) + { + elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", + backup_id_of(backup), encoded_pid); + return 1; + } + + /* unlink shared lock file */ + fio_unlink(lock_file, FIO_BACKUP_HOST); + return 0; +} + +/* + * Lock backup in shared mode + * 0 - successs + * 1 - fail + */ +int +grab_shared_lock_file(pgBackup *backup) +{ + FILE *fp_in = NULL; + FILE *fp_out = NULL; + char buf_in[256]; + pid_t encoded_pid; + char lock_file[MAXPGPATH]; + + char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ + char lock_file_tmp[MAXPGPATH]; + int buffer_len = 0; + + join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); + snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + + /* open already existing lock files */ + fp_in = fopen(lock_file, "r"); + if (fp_in == NULL && errno != ENOENT) + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + + /* read PIDs of owners */ + while (fp_in && fgets(buf_in, sizeof(buf_in), fp_in)) + { + encoded_pid = atoi(buf_in); + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); + continue; + } + + if (encoded_pid == my_pid) + continue; + + if (kill(encoded_pid, 0) == 0) + { + /* + * Somebody is still using this backup in shared mode, + * copy this pid into a new file. + */ + buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } + + if (fp_in) { - atexit(unlink_lock_atexit); - exit_hook_registered = true; + if (ferror(fp_in)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_in); } - /* Use parray so that the lock files are unlinked in a loop */ - if (lock_files == NULL) - lock_files = parray_new(); - parray_append(lock_files, pgut_strdup(lock_file)); + fp_out = fopen(lock_file_tmp, "w"); + if (fp_out == NULL) + { + if (errno == EROFS) + return 0; - return true; + elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + } + + /* add my own pid */ + buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); + + /* write out the collected PIDs to temp lock file */ + fwrite(buffer, 1, buffer_len, fp_out); + + if (ferror(fp_out)) + elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); + + if (fclose(fp_out) != 0) + elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + if (rename(lock_file_tmp, lock_file) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + lock_file_tmp, lock_file, strerror(errno)); + + return 0; +} + +void +unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive) +{ + if (exclusive) + { + release_excl_lock_file(backup_dir); + return; + } + + /* To remove shared lock, we must briefly obtain exclusive lock, ... */ + if (grab_excl_lock_file(backup_dir, backup_id, false) != LOCK_OK) + /* ... if it's not possible then leave shared lock */ + return; + + release_shared_lock_file(backup_dir); + release_excl_lock_file(backup_dir); +} + +void +release_excl_lock_file(const char *backup_dir) +{ + char lock_file[MAXPGPATH]; + + join_path_components(lock_file, backup_dir, BACKUP_LOCK_FILE); + + /* TODO Sanity check: maybe we should check, that pid in lock file is my_pid */ + + /* unlink pid file */ + fio_unlink(lock_file, FIO_BACKUP_HOST); +} + +void +release_shared_lock_file(const char *backup_dir) +{ + FILE *fp_in = NULL; + FILE *fp_out = NULL; + char buf_in[256]; + pid_t encoded_pid; + char lock_file[MAXPGPATH]; + + char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ + char lock_file_tmp[MAXPGPATH]; + int buffer_len = 0; + + join_path_components(lock_file, backup_dir, BACKUP_RO_LOCK_FILE); + snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); + + /* open lock file */ + fp_in = fopen(lock_file, "r"); + if (fp_in == NULL) + { + if (errno == ENOENT) + return; + else + elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); + } + + /* read PIDs of owners */ + while (fgets(buf_in, sizeof(buf_in), fp_in)) + { + encoded_pid = atoi(buf_in); + + if (encoded_pid <= 0) + { + elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); + continue; + } + + /* remove my pid */ + if (encoded_pid == my_pid) + continue; + + if (kill(encoded_pid, 0) == 0) + { + /* + * Somebody is still using this backup in shared mode, + * copy this pid into a new file. + */ + buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); + } + else if (errno != ESRCH) + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + encoded_pid, strerror(errno)); + } + + if (ferror(fp_in)) + elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); + fclose(fp_in); + + /* if there is no active pid left, then there is nothing to do */ + if (buffer_len == 0) + { + fio_unlink(lock_file, FIO_BACKUP_HOST); + return; + } + + fp_out = fopen(lock_file_tmp, "w"); + if (fp_out == NULL) + elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + /* write out the collected PIDs to temp lock file */ + fwrite(buffer, 1, buffer_len, fp_out); + + if (ferror(fp_out)) + elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); + + if (fclose(fp_out) != 0) + elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); + + if (rename(lock_file_tmp, lock_file) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + lock_file_tmp, lock_file, strerror(errno)); + + return; } /* * Get backup_mode in string representation. */ const char * -pgBackupGetBackupMode(pgBackup *backup) +pgBackupGetBackupMode(pgBackup *backup, bool show_color) { - return backupModes[backup->backup_mode]; + if (show_color) + { + /* color the Backup mode */ + char *mode = pgut_malloc(24); /* leaking memory here */ + + if (backup->backup_mode == BACKUP_MODE_FULL) + snprintf(mode, 24, "%s%s%s", TC_GREEN_BOLD, backupModes[backup->backup_mode], TC_RESET); + else + snprintf(mode, 24, "%s%s%s", TC_BLUE_BOLD, backupModes[backup->backup_mode], TC_RESET); + + return mode; + } + else + return backupModes[backup->backup_mode]; } static bool @@ -342,7 +844,7 @@ IsDir(const char *dirpath, const char *entry, fio_location location) char path[MAXPGPATH]; struct stat st; - snprintf(path, MAXPGPATH, "%s/%s", dirpath, entry); + join_path_components(path, dirpath, entry); return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); } @@ -350,13 +852,11 @@ IsDir(const char *dirpath, const char *entry, fio_location location) /* * Create list of instances in given backup catalog. * - * Returns parray of "InstanceConfig" structures, filled with - * actual config of each instance. + * Returns parray of InstanceState structures. */ parray * -catalog_get_instance_list(void) +catalog_get_instance_list(CatalogState *catalogState) { - char path[MAXPGPATH]; DIR *dir; struct dirent *dent; parray *instances; @@ -364,24 +864,23 @@ catalog_get_instance_list(void) instances = parray_new(); /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir = opendir(path); + dir = opendir(catalogState->backup_subdir_path); if (dir == NULL) elog(ERROR, "Cannot open directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); while (errno = 0, (dent = readdir(dir)) != NULL) { char child[MAXPGPATH]; struct stat st; - InstanceConfig *instance; + InstanceState *instanceState = NULL; /* skip entries point current dir or parent dir */ if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0) continue; - join_path_components(child, path, dent->d_name); + join_path_components(child, catalogState->backup_subdir_path, dent->d_name); if (lstat(child, &st) == -1) elog(ERROR, "Cannot stat file \"%s\": %s", @@ -390,9 +889,18 @@ catalog_get_instance_list(void) if (!S_ISDIR(st.st_mode)) continue; - instance = readInstanceConfigFile(dent->d_name); + instanceState = pgut_new(InstanceState); + + strlcpy(instanceState->instance_name, dent->d_name, MAXPGPATH); + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - parray_append(instances, instance); + instanceState->config = readInstanceConfigFile(instanceState); + parray_append(instances, instanceState); } /* TODO 3.0: switch to ERROR */ @@ -401,11 +909,11 @@ catalog_get_instance_list(void) if (errno) elog(ERROR, "Cannot read directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); if (closedir(dir)) elog(ERROR, "Cannot close directory \"%s\": %s", - path, strerror(errno)); + catalogState->backup_subdir_path, strerror(errno)); return instances; } @@ -417,22 +925,18 @@ catalog_get_instance_list(void) * If valid backup id is passed only matching backup will be added to the list. */ parray * -catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) +catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id) { DIR *data_dir = NULL; struct dirent *data_ent = NULL; parray *backups = NULL; int i; - char backup_instance_path[MAXPGPATH]; - - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); /* open backup instance backups directory */ - data_dir = fio_opendir(backup_instance_path, FIO_BACKUP_HOST); + data_dir = fio_opendir(instanceState->instance_backup_subdir_path, FIO_BACKUP_HOST); if (data_dir == NULL) { - elog(WARNING, "cannot open directory \"%s\": %s", backup_instance_path, + elog(WARNING, "cannot open directory \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); goto err_proc; } @@ -446,27 +950,31 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) pgBackup *backup = NULL; /* skip not-directory entries and hidden entries */ - if (!IsDir(backup_instance_path, data_ent->d_name, FIO_BACKUP_HOST) + if (!IsDir(instanceState->instance_backup_subdir_path, data_ent->d_name, FIO_BACKUP_HOST) || data_ent->d_name[0] == '.') continue; /* open subdirectory of specific backup */ - join_path_components(data_path, backup_instance_path, data_ent->d_name); + join_path_components(data_path, instanceState->instance_backup_subdir_path, data_ent->d_name); /* read backup information from BACKUP_CONTROL_FILE */ - snprintf(backup_conf_path, MAXPGPATH, "%s/%s", data_path, BACKUP_CONTROL_FILE); + join_path_components(backup_conf_path, data_path, BACKUP_CONTROL_FILE); backup = readBackupControlFile(backup_conf_path); if (!backup) { - backup = pgut_new(pgBackup); + backup = pgut_new0(pgBackup); pgBackupInit(backup); backup->start_time = base36dec(data_ent->d_name); + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); + backup->backup_id = backup->start_time; } - else if (strcmp(base36enc(backup->start_time), data_ent->d_name) != 0) + else if (strcmp(backup_id_of(backup), data_ent->d_name) != 0) { + /* TODO there is no such guarantees */ elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", - base36enc(backup->start_time), backup_conf_path); + backup_id_of(backup), backup_conf_path); } backup->root_dir = pgut_strdup(data_path); @@ -478,7 +986,6 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) init_header_map(backup); /* TODO: save encoded backup id */ - backup->backup_id = backup->start_time; if (requested_backup_id != INVALID_BACKUP_ID && requested_backup_id != backup->start_time) { @@ -486,18 +993,12 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) continue; } parray_append(backups, backup); - - if (errno && errno != ENOENT) - { - elog(WARNING, "cannot read data directory \"%s\": %s", - data_ent->d_name, strerror(errno)); - goto err_proc; - } } + if (errno) { - elog(WARNING, "cannot read backup root directory \"%s\": %s", - backup_instance_path, strerror(errno)); + elog(WARNING, "Cannot read backup root directory \"%s\": %s", + instanceState->instance_backup_subdir_path, strerror(errno)); goto err_proc; } @@ -511,7 +1012,7 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) { pgBackup *curr = parray_get(backups, i); pgBackup **ancestor; - pgBackup key; + pgBackup key = {0}; if (curr->backup_mode == BACKUP_MODE_FULL) continue; @@ -538,23 +1039,153 @@ catalog_get_backup_list(const char *instance_name, time_t requested_backup_id) } /* - * Create list of backup datafiles. - * If 'requested_backup_id' is INVALID_BACKUP_ID, exit with error. - * If valid backup id is passed only matching backup will be added to the list. - * TODO this function only used once. Is it really needed? + * Get list of files in the backup from the DATABASE_FILE_LIST. */ parray * get_backup_filelist(pgBackup *backup, bool strict) { parray *files = NULL; char backup_filelist_path[MAXPGPATH]; + FILE *fp; + char buf[BLCKSZ]; + char stdio_buf[STDIO_BUFSIZE]; + pg_crc32 content_crc = 0; join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - files = dir_read_file_list(NULL, NULL, backup_filelist_path, FIO_BACKUP_HOST, backup->content_crc); + + fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); + if (fp == NULL) + elog(ERROR, "Cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); + + /* enable stdio buffering for local file */ + if (!fio_is_remote(FIO_BACKUP_HOST)) + setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); + + files = parray_new(); + + INIT_FILE_CRC32(true, content_crc); + + while (fgets(buf, lengthof(buf), fp)) + { + char path[MAXPGPATH]; + char linked[MAXPGPATH]; + char compress_alg_string[MAXPGPATH]; + int64 write_size, + uncompressed_size, + mode, /* bit length of mode_t depends on platforms */ + is_datafile, + is_cfs, + external_dir_num, + crc, + segno, + n_blocks, + n_headers, + dbOid, /* used for partial restore */ + hdr_crc, + hdr_off, + hdr_size; + pgFile *file; + + COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + + get_control_value_str(buf, "path", path, sizeof(path),true); + get_control_value_int64(buf, "size", &write_size, true); + get_control_value_int64(buf, "mode", &mode, true); + get_control_value_int64(buf, "is_datafile", &is_datafile, true); + get_control_value_int64(buf, "is_cfs", &is_cfs, false); + get_control_value_int64(buf, "crc", &crc, true); + get_control_value_str(buf, "compress_alg", compress_alg_string, sizeof(compress_alg_string), false); + get_control_value_int64(buf, "external_dir_num", &external_dir_num, false); + get_control_value_int64(buf, "dbOid", &dbOid, false); + + file = pgFileInit(path); + file->write_size = (int64) write_size; + file->mode = (mode_t) mode; + file->is_datafile = is_datafile ? true : false; + file->is_cfs = is_cfs ? true : false; + file->crc = (pg_crc32) crc; + file->compress_alg = parse_compress_alg(compress_alg_string); + file->external_dir_num = external_dir_num; + file->dbOid = dbOid ? dbOid : 0; + + /* + * Optional fields + */ + if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0]) + { + file->linked = pgut_strdup(linked); + canonicalize_path(file->linked); + } + + if (get_control_value_int64(buf, "segno", &segno, false)) + file->segno = (int) segno; + + if (get_control_value_int64(buf, "n_blocks", &n_blocks, false)) + file->n_blocks = (int) n_blocks; + + if (get_control_value_int64(buf, "n_headers", &n_headers, false)) + file->n_headers = (int) n_headers; + + if (get_control_value_int64(buf, "hdr_crc", &hdr_crc, false)) + file->hdr_crc = (pg_crc32) hdr_crc; + + if (get_control_value_int64(buf, "hdr_off", &hdr_off, false)) + file->hdr_off = hdr_off; + + if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) + file->hdr_size = (int) hdr_size; + + if (get_control_value_int64(buf, "full_size", &uncompressed_size, false)) + file->uncompressed_size = uncompressed_size; + else + file->uncompressed_size = write_size; + if (!file->is_datafile || file->is_cfs) + file->size = file->uncompressed_size; + + if (file->external_dir_num == 0 && + (file->dbOid != 0 || + path_is_prefix_of_path("global", file->rel_path)) && + S_ISREG(file->mode)) + { + bool is_datafile = file->is_datafile; + set_forkname(file); + if (is_datafile != file->is_datafile) + { + if (is_datafile) + elog(WARNING, "File '%s' was stored as datafile, but looks like it is not", + file->rel_path); + else + elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is", + file->rel_path); + /* Lets fail in tests */ + Assert(file->is_datafile == file->is_datafile); + file->is_datafile = is_datafile; + } + } + + parray_append(files, file); + } + + FIN_FILE_CRC32(true, content_crc); + + if (ferror(fp)) + elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); + + fio_close_stream(fp); + + if (backup->content_crc != 0 && + backup->content_crc != content_crc) + { + elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", + backup_filelist_path, content_crc, backup->content_crc); + parray_free(files); + files = NULL; + + } /* redundant sanity? */ if (!files) - elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", base36enc(backup->start_time)); + elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", backup_id_of(backup)); return files; } @@ -563,7 +1194,7 @@ get_backup_filelist(pgBackup *backup, bool strict) * Lock list of backups. Function goes in backward direction. */ void -catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict) +catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict, bool exclusive) { int start_idx, end_idx; @@ -578,9 +1209,9 @@ catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool str for (i = start_idx; i >= end_idx; i--) { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (!lock_backup(backup, strict)) + if (!lock_backup(backup, strict, exclusive)) elog(ERROR, "Cannot lock backup %s directory", - base36enc(backup->start_time)); + backup_id_of(backup)); } } @@ -593,7 +1224,6 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current int i; pgBackup *full_backup = NULL; pgBackup *tmp_backup = NULL; - char *invalid_backup_id; /* backup_list is sorted in order of descending ID */ for (i = 0; i < parray_num(backup_list); i++) @@ -614,7 +1244,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current return NULL; elog(LOG, "Latest valid FULL backup: %s", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* FULL backup is found, lets find his latest child */ for (i = 0; i < parray_num(backup_list); i++) @@ -629,20 +1259,14 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current { /* broken chain */ case ChainIsBroken: - invalid_backup_id = base36enc_dup(tmp_backup->parent_backup); - elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", - base36enc(backup->start_time), invalid_backup_id); - pg_free(invalid_backup_id); + backup_id_of(backup), base36enc(tmp_backup->parent_backup)); continue; /* chain is intact, but at least one parent is invalid */ case ChainIsInvalid: - invalid_backup_id = base36enc_dup(tmp_backup->start_time); - elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", - base36enc(backup->start_time), invalid_backup_id); - pg_free(invalid_backup_id); + backup_id_of(backup), backup_id_of(tmp_backup)); continue; /* chain is ok */ @@ -661,7 +1285,7 @@ catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current else { elog(WARNING, "Backup %s has status: %s. Cannot be a parent.", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); } } @@ -747,7 +1371,7 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; else elog(LOG, "Latest valid full backup: %s, tli: %i", - base36enc(ancestor_backup->start_time), ancestor_backup->tli); + backup_id_of(ancestor_backup), ancestor_backup->tli); /* At this point we found suitable full backup, * now we must find his latest child, suitable to be @@ -812,14 +1436,34 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, return NULL; } -/* create backup directory in $BACKUP_PATH */ -int -pgBackupCreateDir(pgBackup *backup) +/* + * Create backup directory in $BACKUP_PATH + * (with proposed backup->backup_id) + * and initialize this directory. + * If creation of directory fails, then + * backup_id will be cleared (set to INVALID_BACKUP_ID). + * It is possible to get diffrent values in + * pgBackup.start_time and pgBackup.backup_id. + * It may be ok or maybe not, so it's up to the caller + * to fix it or let it be. + */ + +void +pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) { - int i; - char path[MAXPGPATH]; - parray *subdirs = parray_new(); + int i; + char temp[MAXPGPATH]; + parray *subdirs; + /* Try to create backup directory at first */ + if (create_backup_dir(backup, backup_instance_path) != 0) + { + /* Clear backup_id as indication of error */ + reset_backup_id(backup); + return; + } + + subdirs = parray_new(); parray_append(subdirs, pg_strdup(DATABASE_DIR)); /* Add external dirs containers */ @@ -831,7 +1475,6 @@ pgBackupCreateDir(pgBackup *backup) false); for (i = 0; i < parray_num(external_list); i++) { - char temp[MAXPGPATH]; /* Numeration of externaldirs starts with 1 */ makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); parray_append(subdirs, pg_strdup(temp)); @@ -839,14 +1482,6 @@ pgBackupCreateDir(pgBackup *backup) free_dir_list(external_list); } - pgBackupGetPath(backup, path, lengthof(path), NULL); - - if (!dir_is_empty(path, FIO_BACKUP_HOST)) - elog(ERROR, "backup destination is not empty \"%s\"", path); - - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); - backup->root_dir = pgut_strdup(path); - backup->database_dir = pgut_malloc(MAXPGPATH); join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); @@ -856,12 +1491,36 @@ pgBackupCreateDir(pgBackup *backup) /* create directories for actual backup files */ for (i = 0; i < parray_num(subdirs); i++) { - join_path_components(path, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); + join_path_components(temp, backup->root_dir, parray_get(subdirs, i)); + fio_mkdir(temp, DIR_PERMISSION, FIO_BACKUP_HOST); } free_dir_list(subdirs); - return 0; +} + +/* + * Create root directory for backup, + * update pgBackup.root_dir if directory creation was a success + * Return values (same as dir_create_dir()): + * 0 - ok + * -1 - error (warning message already emitted) + */ +int +create_backup_dir(pgBackup *backup, const char *backup_instance_path) +{ + int rc; + char path[MAXPGPATH]; + + join_path_components(path, backup_instance_path, backup_id_of(backup)); + + /* TODO: add wrapper for remote mode */ + rc = dir_create_dir(path, DIR_PERMISSION, true); + + if (rc == 0) + backup->root_dir = pgut_strdup(path); + else + elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); + return rc; } /* @@ -869,22 +1528,21 @@ pgBackupCreateDir(pgBackup *backup) * TODO: '.partial' and '.part' segno information should be added to tlinfo. */ parray * -catalog_get_timelines(InstanceConfig *instance) +catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) { int i,j,k; parray *xlog_files_list = parray_new(); parray *timelineinfos; parray *backups; timelineInfo *tlinfo; - char arclog_path[MAXPGPATH]; /* for fancy reporting */ char begin_segno_str[MAXFNAMELEN]; char end_segno_str[MAXFNAMELEN]; /* read all xlog files that belong to this archive */ - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name); - dir_list_file(xlog_files_list, arclog_path, false, false, false, false, true, 0, FIO_BACKUP_HOST); + dir_list_file(xlog_files_list, instanceState->instance_wal_subdir_path, + false, true, false, false, true, 0, FIO_BACKUP_HOST); parray_qsort(xlog_files_list, pgFileCompareName); timelineinfos = parray_new(); @@ -968,7 +1626,8 @@ catalog_get_timelines(InstanceConfig *instance) } /* temp WAL segment */ else if (IsTempXLogFileName(file->name) || - IsTempCompressXLogFileName(file->name)) + IsTempCompressXLogFileName(file->name) || + IsTempPartialXLogFileName(file->name)) { elog(VERBOSE, "temp WAL file \"%s\"", file->name); @@ -1054,7 +1713,11 @@ catalog_get_timelines(InstanceConfig *instance) TimeLineHistoryEntry *tln; sscanf(file->name, "%08X.history", &tli); - timelines = read_timeline_history(arclog_path, tli, true); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, tli, true); + + /* History file is empty or corrupted, disregard it */ + if (!timelines) + continue; if (!tlinfo || tlinfo->tli != tli) { @@ -1088,7 +1751,7 @@ catalog_get_timelines(InstanceConfig *instance) } /* save information about backups belonging to each timeline */ - backups = catalog_get_backup_list(instance->name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); for (i = 0; i < parray_num(timelineinfos); i++) { @@ -1214,7 +1877,7 @@ catalog_get_timelines(InstanceConfig *instance) { elog(LOG, "Pinned backup %s is ignored for the " "purpose of WAL retention", - base36enc(backup->start_time)); + backup_id_of(backup)); continue; } @@ -1400,7 +2063,7 @@ catalog_get_timelines(InstanceConfig *instance) elog(LOG, "Archive backup %s to stay consistent " "protect from purge WAL interval " "between %s and %s on timeline %i", - base36enc(backup->start_time), + backup_id_of(backup), begin_segno_str, end_segno_str, backup->tli); if (tlinfo->keep_segments == NULL) @@ -1561,7 +2224,7 @@ get_oldest_backup(timelineInfo *tlinfo) * Overwrite backup metadata. */ void -do_set_backup(const char *instance_name, time_t backup_id, +do_set_backup(InstanceState *instanceState, time_t backup_id, pgSetBackupParams *set_backup_params) { pgBackup *target_backup = NULL; @@ -1570,7 +2233,7 @@ do_set_backup(const char *instance_name, time_t backup_id, if (!set_backup_params) elog(ERROR, "Nothing to set by 'set-backup' command"); - backup_list = catalog_get_backup_list(instance_name, backup_id); + backup_list = catalog_get_backup_list(instanceState, backup_id); if (parray_num(backup_list) != 1) elog(ERROR, "Failed to find backup %s", base36enc(backup_id)); @@ -1582,6 +2245,12 @@ do_set_backup(const char *instance_name, time_t backup_id, if (set_backup_params->note) add_note(target_backup, set_backup_params->note); + /* Cleanup */ + if (backup_list) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + } } /* @@ -1595,7 +2264,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) /* sanity, backup must have positive recovery-time */ if (target_backup->recovery_time <= 0) elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'", - base36enc(target_backup->backup_id)); + backup_id_of(target_backup)); /* Pin comes from ttl */ if (set_backup_params->ttl > 0) @@ -1609,7 +2278,7 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) if (target_backup->expire_time == 0) { elog(WARNING, "Backup %s is not pinned, nothing to unpin", - base36enc(target_backup->start_time)); + backup_id_of(target_backup)); return; } target_backup->expire_time = 0; @@ -1628,12 +2297,12 @@ pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) { char expire_timestamp[100]; - time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time); - elog(INFO, "Backup %s is pinned until '%s'", base36enc(target_backup->start_time), + time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time, false); + elog(INFO, "Backup %s is pinned until '%s'", backup_id_of(target_backup), expire_timestamp); } else - elog(INFO, "Backup %s is unpinned", base36enc(target_backup->start_time)); + elog(INFO, "Backup %s is unpinned", backup_id_of(target_backup)); return; } @@ -1647,13 +2316,14 @@ add_note(pgBackup *target_backup, char *note) { char *note_string; + char *p; /* unset note */ if (pg_strcasecmp(note, "none") == 0) { target_backup->note = NULL; elog(INFO, "Removing note from backup %s", - base36enc(target_backup->start_time)); + backup_id_of(target_backup)); } else { @@ -1663,12 +2333,12 @@ add_note(pgBackup *target_backup, char *note) * we save only "aaa" * Example: tests.set_backup.SetBackupTest.test_add_note_newlines */ - note_string = pgut_malloc(MAX_NOTE_SIZE); - sscanf(note, "%[^\n]", note_string); + p = strchr(note, '\n'); + note_string = pgut_strndup(note, p ? (p-note) : MAX_NOTE_SIZE); target_backup->note = note_string; elog(INFO, "Adding note to backup %s: '%s'", - base36enc(target_backup->start_time), target_backup->note); + backup_id_of(target_backup), target_backup->note); } /* Update backup.control */ @@ -1679,12 +2349,12 @@ add_note(pgBackup *target_backup, char *note) * Write information about backup.in to stream "out". */ void -pgBackupWriteControl(FILE *out, pgBackup *backup) +pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) { char timestamp[100]; fio_fprintf(out, "#Configuration\n"); - fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup)); + fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup, false)); fio_fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); fio_fprintf(out, "compress-alg = %s\n", deparse_compress_alg(backup->compress_alg)); @@ -1711,27 +2381,27 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, utc); fio_fprintf(out, "start-time = '%s'\n", timestamp); if (backup->merge_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->merge_time); + time2iso(timestamp, lengthof(timestamp), backup->merge_time, utc); fio_fprintf(out, "merge-time = '%s'\n", timestamp); } if (backup->end_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, utc); fio_fprintf(out, "end-time = '%s'\n", timestamp); } fio_fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, utc); fio_fprintf(out, "recovery-time = '%s'\n", timestamp); } if (backup->expire_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->expire_time); + time2iso(timestamp, lengthof(timestamp), backup->expire_time, utc); fio_fprintf(out, "expire-time = '%s'\n", timestamp); } @@ -1778,7 +2448,9 @@ pgBackupWriteControl(FILE *out, pgBackup *backup) /* * Save the backup content into BACKUP_CONTROL_FILE. - * TODO: honor the strict flag + * Flag strict allows to ignore "out of space" error + * when attempting to lock backup. Only delete is allowed + * to use this functionality. */ void write_backup(pgBackup *backup, bool strict) @@ -1786,7 +2458,7 @@ write_backup(pgBackup *backup, bool strict) FILE *fp = NULL; char path[MAXPGPATH]; char path_temp[MAXPGPATH]; - char buf[4096]; + char buf[8192]; join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); @@ -1802,20 +2474,36 @@ write_backup(pgBackup *backup, bool strict) setvbuf(fp, buf, _IOFBF, sizeof(buf)); - pgBackupWriteControl(fp, backup); + pgBackupWriteControl(fp, backup, true); + /* Ignore 'out of space' error in lax mode */ if (fflush(fp) != 0) - elog(ERROR, "Cannot flush control file \"%s\": %s", - path_temp, strerror(errno)); + { + int elevel = ERROR; + int save_errno = errno; - if (fsync(fileno(fp)) < 0) - elog(ERROR, "Cannot sync control file \"%s\": %s", - path_temp, strerror(errno)); + if (!strict && (errno == ENOSPC)) + elevel = WARNING; + + elog(elevel, "Cannot flush control file \"%s\": %s", + path_temp, strerror(save_errno)); + + if (!strict && (save_errno == ENOSPC)) + { + fclose(fp); + fio_unlink(path_temp, FIO_BACKUP_HOST); + return; + } + } if (fclose(fp) != 0) elog(ERROR, "Cannot close control file \"%s\": %s", path_temp, strerror(errno)); + if (fio_sync(path_temp, FIO_BACKUP_HOST) < 0) + elog(ERROR, "Cannot sync control file \"%s\": %s", + path_temp, strerror(errno)); + if (rename(path_temp, path) < 0) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", path_temp, path, strerror(errno)); @@ -1832,7 +2520,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, char control_path[MAXPGPATH]; char control_path_temp[MAXPGPATH]; size_t i = 0; - #define BUFFERSZ 1024*1024 + #define BUFFERSZ (1024*1024) char *buf; int64 backup_size_on_disk = 0; int64 uncompressed_size_on_disk = 0; @@ -1880,7 +2568,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, * Size of WAL files in 'pg_wal' is counted separately * TODO: in 3.0 add attribute is_walfile */ - if (IsXLogFileName(file->name) && (file->external_dir_num == 0)) + if (IsXLogFileName(file->name) && file->external_dir_num == 0) wal_size_on_disk += file->write_size; else { @@ -1902,6 +2590,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, file->external_dir_num, file->dbOid); + if (file->uncompressed_size != 0 && + file->uncompressed_size != file->write_size) + len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"", + file->uncompressed_size); + if (file->is_datafile) len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); @@ -1915,7 +2608,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, { len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); - len += sprintf(line+len, ",\"hdr_off\":\"%li\"", file->hdr_off); + len += sprintf(line+len, ",\"hdr_off\":\"%llu\"", file->hdr_off); len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size); } @@ -1964,7 +2657,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, static pgBackup * readBackupControlFile(const char *path) { - pgBackup *backup = pgut_new(pgBackup); + pgBackup *backup = pgut_new0(pgBackup); char *backup_mode = NULL; char *start_lsn = NULL; char *stop_lsn = NULL; @@ -2034,6 +2727,9 @@ readBackupControlFile(const char *path) pgBackupFree(backup); return NULL; } + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); + backup->backup_id = backup->start_time; if (backup_mode) { @@ -2106,14 +2802,14 @@ readBackupControlFile(const char *path) if (program_version) { - StrNCpy(backup->program_version, program_version, + strlcpy(backup->program_version, program_version, sizeof(backup->program_version)); pfree(program_version); } if (server_version) { - StrNCpy(backup->server_version, server_version, + strlcpy(backup->server_version, server_version, sizeof(backup->server_version)); pfree(server_version); } @@ -2145,7 +2841,7 @@ parse_backup_mode(const char *value) return BACKUP_MODE_DIFF_DELTA; /* Backup mode is invalid, so leave with an error */ - elog(ERROR, "invalid backup-mode \"%s\"", value); + elog(ERROR, "Invalid backup-mode \"%s\"", value); return BACKUP_MODE_INVALID; } @@ -2180,7 +2876,7 @@ parse_compress_alg(const char *arg) len = strlen(arg); if (len == 0) - elog(ERROR, "compress algorithm is empty"); + elog(ERROR, "Compress algorithm is empty"); if (pg_strncasecmp("zlib", arg, len) == 0) return ZLIB_COMPRESS; @@ -2189,7 +2885,7 @@ parse_compress_alg(const char *arg) else if (pg_strncasecmp("none", arg, len) == 0) return NONE_COMPRESS; else - elog(ERROR, "invalid compress algorithm value \"%s\"", arg); + elog(ERROR, "Invalid compress algorithm value \"%s\"", arg); return NOT_DEFINED_COMPRESS; } @@ -2228,7 +2924,7 @@ pgNodeInit(PGNodeInfo *node) node->server_version_str[0] = '\0'; node->ptrack_version_num = 0; - node->is_ptrack_enable = false; + node->is_ptrack_enabled = false; node->ptrack_schema = NULL; } @@ -2315,64 +3011,6 @@ pgBackupCompareIdDesc(const void *l, const void *r) return -pgBackupCompareId(l, r); } -/* - * Construct absolute path of the backup directory. - * If subdir is not NULL, it will be appended after the path. - */ -void -pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subdir) -{ - pgBackupGetPath2(backup, path, len, subdir, NULL); -} - -/* - * Construct absolute path of the backup directory. - * Append "subdir1" and "subdir2" to the backup directory. - */ -void -pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2) -{ - /* If "subdir1" is NULL do not check "subdir2" */ - if (!subdir1) - snprintf(path, len, "%s/%s", backup_instance_path, - base36enc(backup->start_time)); - else if (!subdir2) - snprintf(path, len, "%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1); - /* "subdir1" and "subdir2" is not NULL */ - else - snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1, subdir2); -} - -/* - * independent from global variable backup_instance_path - * Still depends from backup_path - */ -void -pgBackupGetPathInInstance(const char *instance_name, - const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2) -{ - char backup_instance_path[MAXPGPATH]; - - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - - /* If "subdir1" is NULL do not check "subdir2" */ - if (!subdir1) - snprintf(path, len, "%s/%s", backup_instance_path, - base36enc(backup->start_time)); - else if (!subdir2) - snprintf(path, len, "%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1); - /* "subdir1" and "subdir2" is not NULL */ - else - snprintf(path, len, "%s/%s/%s/%s", backup_instance_path, - base36enc(backup->start_time), subdir1, subdir2); -} - /* * Check if multiple backups consider target backup to be their direct parent */ @@ -2425,7 +3063,7 @@ find_parent_full_backup(pgBackup *current_backup) base36enc(base_full_backup->parent_backup)); else elog(WARNING, "Failed to find parent FULL backup for %s", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); return NULL; } @@ -2522,26 +3160,6 @@ is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive) return false; } -/* - * Return backup index number. - * Note: this index number holds true until new sorting of backup list - */ -int -get_backup_index_number(parray *backup_list, pgBackup *backup) -{ - int i; - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); - - if (tmp_backup->start_time == backup->start_time) - return i; - } - elog(WARNING, "Failed to find backup %s", base36enc(backup->start_time)); - return -1; -} - /* On backup_list lookup children of target_backup and append them to append_list */ void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list) diff --git a/src/catchup.c b/src/catchup.c new file mode 100644 index 000000000..00752b194 --- /dev/null +++ b/src/catchup.c @@ -0,0 +1,1170 @@ +/*------------------------------------------------------------------------- + * + * catchup.c: sync DB cluster + * + * Copyright (c) 2021-2022, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" + +#if PG_VERSION_NUM < 110000 +#include "catalog/catalog.h" +#endif +#include "catalog/pg_tablespace.h" +#include "access/timeline.h" +#include "pgtar.h" +#include "streamutil.h" + +#include +#include +#include + +#include "utils/thread.h" +#include "utils/file.h" + +/* + * Catchup routines + */ +static PGconn *catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); +static void catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, + const char *dest_pgdata); +static void catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn); +static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli); + +//REVIEW I'd also suggest to wrap all these fields into some CatchupState, but it isn't urgent. +//REVIEW_ANSWER what for? +/* + * Prepare for work: fill some globals, open connection to source database + */ +static PGconn * +catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) +{ + PGconn *source_conn; + + /* Initialize PGInfonode */ + pgNodeInit(source_node_info); + + /* Get WAL segments size and system ID of source PG instance */ + instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); + instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false); + current.start_time = time(NULL); + + strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); + + /* Do some compatibility checks and fill basic info about PG instance */ + source_conn = pgdata_basic_setup(instance_config.conn_opt, source_node_info); + +#if PG_VERSION_NUM >= 110000 + if (!RetrieveWalSegSize(source_conn)) + elog(ERROR, "Failed to retrieve wal_segment_size"); +#endif + + get_ptrack_version(source_conn, source_node_info); + if (source_node_info->ptrack_version_num > 0) + source_node_info->is_ptrack_enabled = pg_is_ptrack_enabled(source_conn, source_node_info->ptrack_version_num); + + /* Obtain current timeline */ +#if PG_VERSION_NUM >= 90600 + current.tli = get_current_timeline(source_conn); +#else + instance_config.pgdata = source_pgdata; + current.tli = get_current_timeline_from_control(source_pgdata, FIO_DB_HOST, false); +#endif + + elog(INFO, "Catchup start, pg_probackup version: %s, " + "PostgreSQL version: %s, " + "remote: %s, source-pgdata: %s, destination-pgdata: %s", + PROGRAM_VERSION, source_node_info->server_version_str, + IsSshProtocol() ? "true" : "false", + source_pgdata, dest_pgdata); + + if (current.from_replica) + elog(INFO, "Running catchup from standby"); + + return source_conn; +} + +/* + * Check that catchup can be performed on source and dest + * this function is for checks, that can be performed without modification of data on disk + */ +static void +catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, + const char *source_pgdata, const char *dest_pgdata) +{ + /* TODO + * gsmol - fallback to FULL mode if dest PGDATA is empty + * kulaginm -- I think this is a harmful feature. If user requested an incremental catchup, then + * he expects that this will be done quickly and efficiently. If, for example, he made a mistake + * with dest_dir, then he will receive a second full copy instead of an error message, and I think + * that in some cases he would prefer the error. + * I propose in future versions to offer a backup_mode auto, in which we will look to the dest_dir + * and decide which of the modes will be the most effective. + * I.e.: + * if(requested_backup_mode == BACKUP_MODE_DIFF_AUTO) + * { + * if(dest_pgdata_is_empty) + * backup_mode = BACKUP_MODE_FULL; + * else + * if(ptrack supported and applicable) + * backup_mode = BACKUP_MODE_DIFF_PTRACK; + * else + * backup_mode = BACKUP_MODE_DIFF_DELTA; + * } + */ + + if (dir_is_empty(dest_pgdata, FIO_LOCAL_HOST)) + { + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK || + current.backup_mode == BACKUP_MODE_DIFF_DELTA) + elog(ERROR, "\"%s\" is empty, but incremental catchup mode requested.", + dest_pgdata); + } + else /* dest dir not empty */ + { + if (current.backup_mode == BACKUP_MODE_FULL) + elog(ERROR, "Can't perform full catchup into non-empty directory \"%s\".", + dest_pgdata); + } + + /* check that postmaster is not running in destination */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + pid_t pid; + pid = fio_check_postmaster(dest_pgdata, FIO_LOCAL_HOST); + if (pid == 1) /* postmaster.pid is mangled */ + { + char pid_filename[MAXPGPATH]; + join_path_components(pid_filename, dest_pgdata, "postmaster.pid"); + elog(ERROR, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", + pid_filename); + } + else if (pid > 1) /* postmaster is up */ + { + elog(ERROR, "Postmaster with pid %u is running in destination directory \"%s\"", + pid, dest_pgdata); + } + } + + /* check backup_label absence in dest */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + char backup_label_filename[MAXPGPATH]; + + join_path_components(backup_label_filename, dest_pgdata, PG_BACKUP_LABEL_FILE); + if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0) + elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); + } + + /* Check that connected PG instance, source and destination PGDATA are the same */ + { + uint64 source_conn_id, source_id, dest_id; + + source_conn_id = get_remote_system_identifier(source_conn); + source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */ + + if (source_conn_id != source_id) + elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", + source_conn_id, source_pgdata, source_id); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + ControlFileData dst_control; + get_control_file_or_back_file(dest_pgdata, FIO_LOCAL_HOST, &dst_control); + dest_id = dst_control.system_identifier; + + if (source_conn_id != dest_id) + elog(ERROR, "Database identifiers mismatch: we connected to DB id %llu, but in \"%s\" we found id %llu", + (long long)source_conn_id, dest_pgdata, (long long)dest_id); + } + } + + /* check PTRACK version */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + if (source_node_info->ptrack_version_num == 0) + elog(ERROR, "This PostgreSQL instance does not support ptrack"); + else if (source_node_info->ptrack_version_num < 200) + elog(ERROR, "Ptrack extension is too old.\n" + "Upgrade ptrack to version >= 2"); + else if (!source_node_info->is_ptrack_enabled) + elog(ERROR, "Ptrack is disabled"); + } + + if (current.from_replica && exclusive_backup) + elog(ERROR, "Catchup from standby is only available for PostgreSQL >= 9.6"); + + /* check that we don't overwrite tablespace in source pgdata */ + catchup_check_tablespaces_existance_in_tbsmapping(source_conn); + + /* check timelines */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; + + /* fill dest_redo.lsn and dest_redo.tli */ + get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + elog(LOG, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", + current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); + + if (current.tli != 1) + { + parray *source_timelines; /* parray* of TimeLineHistoryEntry* */ + source_timelines = catchup_get_tli_history(&instance_config.conn_opt, current.tli); + + if (source_timelines == NULL) + elog(ERROR, "Cannot get source timeline history"); + + if (!satisfy_timeline(source_timelines, dest_redo.tli, dest_redo.lsn)) + elog(ERROR, "Destination is not in source timeline history"); + + parray_walk(source_timelines, pfree); + parray_free(source_timelines); + } + else /* special case -- no history files in source */ + { + if (dest_redo.tli != 1) + elog(ERROR, "Source is behind destination in timeline history"); + } + } +} + +/* + * Check that all tablespaces exists in tablespace mapping (--tablespace-mapping option) + * Check that all local mapped directories is empty if it is local FULL catchup + * Emit fatal error if that (not existent in map or not empty) tablespace found + */ +static void +catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) +{ + PGresult *res; + int i; + char *tablespace_path = NULL; + const char *linked_path = NULL; + char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " + "FROM pg_catalog.pg_tablespace " + "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; + + res = pgut_execute(conn, query, 0, NULL); + + if (!res) + elog(ERROR, "Failed to get list of tablespaces"); + + for (i = 0; i < res->ntups; i++) + { + tablespace_path = PQgetvalue(res, i, 0); + Assert (strlen(tablespace_path) > 0); + + canonicalize_path(tablespace_path); + linked_path = get_tablespace_mapping(tablespace_path); + + if (strcmp(tablespace_path, linked_path) == 0) + /* same result -> not found in mapping */ + { + if (!fio_is_remote(FIO_DB_HOST)) + elog(ERROR, "Local catchup executed, but source database contains " + "tablespace (\"%s\"), that is not listed in the map", tablespace_path); + else + elog(WARNING, "Remote catchup executed and source database contains " + "tablespace (\"%s\"), that is not listed in the map", tablespace_path); + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "Tablespace directory path must be an absolute path: \"%s\"", + linked_path); + + if (current.backup_mode == BACKUP_MODE_FULL + && !dir_is_empty(linked_path, FIO_LOCAL_HOST)) + elog(ERROR, "Target mapped tablespace directory (\"%s\") is not empty in FULL catchup", + linked_path); + } + PQclear(res); +} + +/* + * Get timeline history via replication connection + * returns parray* of TimeLineHistoryEntry* + */ +static parray* +catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli) +{ + PGresult *res; + PGconn *conn; + char *history; + char query[128]; + parray *result = NULL; + TimeLineHistoryEntry *entry = NULL; + + snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); + + /* + * Connect in replication mode to the server. + */ + conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser, + false); + + if (!conn) + return NULL; + + res = PQexec(conn, query); + PQfinish(conn); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING, "Could not send replication command \"%s\": %s", + query, PQresultErrorMessage(res)); + PQclear(res); + return NULL; + } + + /* + * The response to TIMELINE_HISTORY is a single row result set + * with two fields: filename and content + */ + if (PQnfields(res) != 2 || PQntuples(res) != 1) + { + elog(ERROR, "Unexpected response to TIMELINE_HISTORY command: " + "got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + PQclear(res); + return NULL; + } + + history = pgut_strdup(PQgetvalue(res, 0, 1)); + result = parse_tli_history_buffer(history, tli); + + /* some cleanup */ + pg_free(history); + PQclear(res); + + /* append last timeline entry (as read_timeline_history() do) */ + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = InvalidXLogRecPtr; + parray_insert(result, 0, entry); + + return result; +} + +/* + * catchup multithreaded copy rountine and helper structure and function + */ + +/* parameters for catchup_thread_runner() passed from catchup_multithreaded_copy() */ +typedef struct +{ + PGNodeInfo *nodeInfo; + const char *from_root; + const char *to_root; + parray *source_filelist; + parray *dest_filelist; + XLogRecPtr sync_lsn; + BackupMode backup_mode; + int thread_num; + size_t transfered_bytes; + bool completed; +} catchup_thread_runner_arg; + +/* Catchup file copier executed in separate thread */ +static void * +catchup_thread_runner(void *arg) +{ + int i; + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + + catchup_thread_runner_arg *arguments = (catchup_thread_runner_arg *) arg; + int n_files = parray_num(arguments->source_filelist); + + /* catchup a file */ + for (i = 0; i < n_files; i++) + { + pgFile *file = (pgFile *) parray_get(arguments->source_filelist, i); + pgFile *dest_file = NULL; + + /* We have already copied all directories */ + if (S_ISDIR(file->mode)) + continue; + + if (file->excluded) + continue; + + if (!pg_atomic_test_set_flag(&file->lock)) + continue; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during catchup"); + + elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", + i + 1, n_files, file->rel_path); + + /* construct destination filepath */ + Assert(file->external_dir_num == 0); + join_path_components(from_fullpath, arguments->from_root, file->rel_path); + join_path_components(to_fullpath, arguments->to_root, file->rel_path); + + /* Encountered some strange beast */ + if (!S_ISREG(file->mode)) + elog(WARNING, "Unexpected type %d of file \"%s\", skipping", + file->mode, from_fullpath); + + /* Check that file exist in dest pgdata */ + if (arguments->backup_mode != BACKUP_MODE_FULL) + { + pgFile **dest_file_tmp = NULL; + dest_file_tmp = (pgFile **) parray_bsearch(arguments->dest_filelist, + file, pgFileCompareRelPathWithExternal); + if (dest_file_tmp) + { + /* File exists in destination PGDATA */ + file->exists_in_prev = true; + dest_file = *dest_file_tmp; + } + } + + /* Do actual work */ + if (file->is_datafile && !file->is_cfs) + { + catchup_data_file(file, from_fullpath, to_fullpath, + arguments->sync_lsn, + arguments->backup_mode, + arguments->nodeInfo->checksum_version, + dest_file != NULL ? dest_file->size : 0); + } + else + { + backup_non_data_file(file, dest_file, from_fullpath, to_fullpath, + arguments->backup_mode, current.parent_backup, true); + } + + /* file went missing during catchup */ + if (file->write_size == FILE_NOT_FOUND) + continue; + + if (file->write_size == BYTES_INVALID) + { + elog(LOG, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); + continue; + } + + arguments->transfered_bytes += file->write_size; + elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", + from_fullpath, file->write_size); + } + + /* ssh connection to longer needed */ + fio_disconnect(); + + /* Data files transferring is successful */ + arguments->completed = true; + + return NULL; +} + +/* + * main multithreaded copier + * returns size of transfered data file + * or -1 in case of error + */ +static ssize_t +catchup_multithreaded_copy(int num_threads, + PGNodeInfo *source_node_info, + const char *source_pgdata_path, + const char *dest_pgdata_path, + parray *source_filelist, + parray *dest_filelist, + XLogRecPtr sync_lsn, + BackupMode backup_mode) +{ + /* arrays with meta info for multi threaded catchup */ + catchup_thread_runner_arg *threads_args; + pthread_t *threads; + + bool all_threads_successful = true; + ssize_t transfered_bytes_result = 0; + int i; + + /* init thread args */ + threads_args = (catchup_thread_runner_arg *) palloc(sizeof(catchup_thread_runner_arg) * num_threads); + for (i = 0; i < num_threads; i++) + threads_args[i] = (catchup_thread_runner_arg){ + .nodeInfo = source_node_info, + .from_root = source_pgdata_path, + .to_root = dest_pgdata_path, + .source_filelist = source_filelist, + .dest_filelist = dest_filelist, + .sync_lsn = sync_lsn, + .backup_mode = backup_mode, + .thread_num = i + 1, + .transfered_bytes = 0, + .completed = false, + }; + + /* Run threads */ + thread_interrupted = false; + threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); + if (!dry_run) + { + for (i = 0; i < num_threads; i++) + { + elog(VERBOSE, "Start thread num: %i", i); + pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i])); + } + } + + /* Wait threads */ + for (i = 0; i < num_threads; i++) + { + if (!dry_run) + pthread_join(threads[i], NULL); + all_threads_successful &= threads_args[i].completed; + transfered_bytes_result += threads_args[i].transfered_bytes; + } + + free(threads); + free(threads_args); + return all_threads_successful ? transfered_bytes_result : -1; +} + +/* + * Sync every file in destination directory to disk + */ +static void +catchup_sync_destination_files(const char* pgdata_path, fio_location location, parray *filelist, pgFile *pg_control_file) +{ + char fullpath[MAXPGPATH]; + time_t start_time, end_time; + char pretty_time[20]; + int i; + + elog(INFO, "Syncing copied files to disk"); + time(&start_time); + + for (i = 0; i < parray_num(filelist); i++) + { + pgFile *file = (pgFile *) parray_get(filelist, i); + + /* TODO: sync directory ? + * - at first glance we can rely on fs journaling, + * which is enabled by default on most platforms + * - but PG itself is not relying on fs, its durable_sync + * includes directory sync + */ + if (S_ISDIR(file->mode) || file->excluded) + continue; + + Assert(file->external_dir_num == 0); + join_path_components(fullpath, pgdata_path, file->rel_path); + if (fio_sync(fullpath, location) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); + } + + /* + * sync pg_control file + */ + join_path_components(fullpath, pgdata_path, pg_control_file->rel_path); + if (fio_sync(fullpath, location) != 0) + elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + elog(INFO, "Files are synced, time elapsed: %s", pretty_time); +} + +/* + * Filter filelist helper function (used to process --exclude-path's) + * filelist -- parray of pgFile *, can't be NULL + * exclude_absolute_paths_list -- sorted parray of char * (absolute paths, starting with '/'), can be NULL + * exclude_relative_paths_list -- sorted parray of char * (relative paths), can be NULL + * logging_string -- helper parameter, used for generating verbose log messages ("Source" or "Destination") + */ +static void +filter_filelist(parray *filelist, const char *pgdata, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list, + const char *logging_string) +{ + int i; + + if (exclude_absolute_paths_list == NULL && exclude_relative_paths_list == NULL) + return; + + for (i = 0; i < parray_num(filelist); ++i) + { + char full_path[MAXPGPATH]; + pgFile *file = (pgFile *) parray_get(filelist, i); + join_path_components(full_path, pgdata, file->rel_path); + + if ( + (exclude_absolute_paths_list != NULL + && parray_bsearch(exclude_absolute_paths_list, full_path, pgPrefixCompareString)!= NULL + ) || ( + exclude_relative_paths_list != NULL + && parray_bsearch(exclude_relative_paths_list, file->rel_path, pgPrefixCompareString)!= NULL) + ) + { + elog(INFO, "%s file \"%s\" excluded with --exclude-path option", logging_string, full_path); + file->excluded = true; + } + } +} + +/* + * Entry point of pg_probackup CATCHUP subcommand. + * exclude_*_paths_list are parray's of char * + */ +int +do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list) +{ + PGconn *source_conn = NULL; + PGNodeInfo source_node_info; + bool backup_logs = false; + parray *source_filelist = NULL; + pgFile *source_pg_control_file = NULL; + parray *dest_filelist = NULL; + char dest_xlog_path[MAXPGPATH]; + + RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; + PGStopBackupResult stop_backup_result; + bool catchup_isok = true; + + int i; + + /* for fancy reporting */ + time_t start_time, end_time; + ssize_t transfered_datafiles_bytes = 0; + ssize_t transfered_walfiles_bytes = 0; + char pretty_source_bytes[20]; + + char dest_pg_control_fullpath[MAXPGPATH]; + char dest_pg_control_bak_fullpath[MAXPGPATH]; + + source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata); + catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); + + /* we need to sort --exclude_path's for future searching */ + if (exclude_absolute_paths_list != NULL) + parray_qsort(exclude_absolute_paths_list, pgCompareString); + if (exclude_relative_paths_list != NULL) + parray_qsort(exclude_relative_paths_list, pgCompareString); + + elog(INFO, "Database catchup start"); + + if (current.backup_mode != BACKUP_MODE_FULL) + { + dest_filelist = parray_new(); + dir_list_file(dest_filelist, dest_pgdata, + true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination"); + + // fill dest_redo.lsn and dest_redo.tli + get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn); + + /* + * Future improvement to catch partial catchup: + * 1. rename dest pg_control into something like pg_control.pbk + * (so user can't start partial catchup'ed instance from this point) + * 2. try to read by get_redo() pg_control and pg_control.pbk (to detect partial catchup) + * 3. at the end (after copy of correct pg_control), remove pg_control.pbk + */ + } + + /* + * Make sure that sync point is withing ptrack tracking range + * TODO: move to separate function to use in both backup.c and catchup.c + */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, &source_node_info); + + if (ptrack_lsn > dest_redo.lsn || ptrack_lsn == InvalidXLogRecPtr) + elog(ERROR, "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" + "You can perform only FULL catchup.", + (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), + (uint32) (dest_redo.lsn >> 32), + (uint32) (dest_redo.lsn)); + } + + { + char label[1024]; + /* notify start of backup to PostgreSQL server */ + time2iso(label, lengthof(label), current.start_time, false); + strncat(label, " with pg_probackup", lengthof(label) - + strlen(" with pg_probackup")); + + /* Call pg_start_backup function in PostgreSQL connect */ + pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); + elog(INFO, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); + } + + /* Sanity: source cluster must be "in future" relatively to dest cluster */ + if (current.backup_mode != BACKUP_MODE_FULL && + dest_redo.lsn > current.start_lsn) + elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " + "it may indicate that we are trying to catchup with PostgreSQL instance from the past", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn)); + + /* Start stream replication */ + join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); + if (!dry_run) + { + fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); + start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, + current.start_lsn, current.tli, false); + } + else + elog(INFO, "WAL streaming skipping with --dry-run option"); + + source_filelist = parray_new(); + + /* list files with the logical path. omit $PGDATA */ + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir(source_filelist, source_pgdata, + true, true, false, backup_logs, true, 0); + else + dir_list_file(source_filelist, source_pgdata, + true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); + + //REVIEW FIXME. Let's fix that before release. + // TODO what if wal is not a dir (symlink to a dir)? + // - Currently backup/restore transform pg_wal symlink to directory + // so the problem is not only with catchup. + // if we want to make it right - we must provide the way + // for symlink remapping during restore and catchup. + // By default everything must be left as it is. + + /* close ssh session in main thread */ + fio_disconnect(); + + /* + * Sort pathname ascending. It is necessary to create intermediate + * directories sequentially. + * + * For example: + * 1 - create 'base' + * 2 - create 'base/1' + * + * Sorted array is used at least in parse_filelist_filenames(), + * extractPageMap(), make_pagemap_from_ptrack(). + */ + parray_qsort(source_filelist, pgFileCompareRelPathWithExternal); + + //REVIEW Do we want to do similar calculation for dest? + //REVIEW_ANSWER what for? + { + ssize_t source_bytes = 0; + char pretty_bytes[20]; + + source_bytes += calculate_datasize_of_filelist(source_filelist); + + /* Extract information about files in source_filelist parsing their names:*/ + parse_filelist_filenames(source_filelist, source_pgdata); + filter_filelist(source_filelist, source_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Source"); + + current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); + + pretty_size(current.pgdata_bytes, pretty_source_bytes, lengthof(pretty_source_bytes)); + pretty_size(source_bytes - current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); + elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, pretty_bytes); + } + + elog(INFO, "Start LSN (source): %X/%X, TLI: %X", + (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), + current.tli); + if (current.backup_mode != BACKUP_MODE_FULL) + elog(INFO, "LSN in destination: %X/%X, TLI: %X", + (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn), + dest_redo.tli); + + /* Build page mapping in PTRACK mode */ + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + time(&start_time); + elog(INFO, "Extracting pagemap of changed blocks"); + + /* Build the page map from ptrack information */ + make_pagemap_from_ptrack_2(source_filelist, source_conn, + source_node_info.ptrack_schema, + source_node_info.ptrack_version_num, + dest_redo.lsn); + time(&end_time); + elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", + difftime(end_time, start_time)); + } + + /* + * Make directories before catchup + */ + /* + * We iterate over source_filelist and for every directory with parent 'pg_tblspc' + * we must lookup this directory name in tablespace map. + * If we got a match, we treat this directory as tablespace. + * It means that we create directory specified in tablespace map and + * original directory created as symlink to it. + */ + for (i = 0; i < parray_num(source_filelist); i++) + { + pgFile *file = (pgFile *) parray_get(source_filelist, i); + char parent_dir[MAXPGPATH]; + + if (!S_ISDIR(file->mode) || file->excluded) + continue; + + /* + * check if it is fake "directory" and is a tablespace link + * this is because we passed the follow_symlink when building the list + */ + /* get parent dir of rel_path */ + strncpy(parent_dir, file->rel_path, MAXPGPATH); + get_parent_directory(parent_dir); + + /* check if directory is actually link to tablespace */ + if (strcmp(parent_dir, PG_TBLSPC_DIR) != 0) + { + /* if the entry is a regular directory, create it in the destination */ + char dirpath[MAXPGPATH]; + + join_path_components(dirpath, dest_pgdata, file->rel_path); + + elog(LOG, "Create directory '%s'", dirpath); + if (!dry_run) + fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); + } + else + { + /* this directory located in pg_tblspc */ + const char *linked_path = NULL; + char to_path[MAXPGPATH]; + + // TODO perform additional check that this is actually symlink? + { /* get full symlink path and map this path to new location */ + char source_full_path[MAXPGPATH]; + char symlink_content[MAXPGPATH]; + join_path_components(source_full_path, source_pgdata, file->rel_path); + fio_readlink(source_full_path, symlink_content, sizeof(symlink_content), FIO_DB_HOST); + /* we checked that mapping exists in preflight_checks for local catchup */ + linked_path = get_tablespace_mapping(symlink_content); + elog(INFO, "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", + source_full_path, + symlink_content, + linked_path); + } + + if (!is_absolute_path(linked_path)) + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", + linked_path); + + join_path_components(to_path, dest_pgdata, file->rel_path); + + elog(INFO, "Create directory \"%s\" and symbolic link \"%s\"", + linked_path, to_path); + + if (!dry_run) + { + /* create tablespace directory */ + if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) + elog(ERROR, "Could not create tablespace directory \"%s\": %s", + linked_path, strerror(errno)); + + /* create link to linked_path */ + if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) + elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", + linked_path, to_path, strerror(errno)); + } + } + } + + /* + * find pg_control file (in already sorted source_filelist) + * and exclude it from list for future special processing + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(source_filelist, &search_key, pgFileCompareRelPathWithExternal); + if(control_file_elem_index < 0) + elog(ERROR, "\"%s\" not found in \"%s\"\n", XLOG_CONTROL_FILE, source_pgdata); + source_pg_control_file = parray_remove(source_filelist, control_file_elem_index); + } + + /* TODO before public release: must be more careful with pg_control. + * when running catchup or incremental restore + * cluster is actually in two states + * simultaneously - old and new, so + * it must contain both pg_control files + * describing those states: global/pg_control_old, global/pg_control_new + * 1. This approach will provide us with means of + * robust detection of previos failures and thus correct operation retrying (or forbidding). + * 2. We will have the ability of preventing instance from starting + * in the middle of our operations. + */ + + /* + * remove absent source files in dest (dropped tables, etc...) + * note: global/pg_control will also be deleted here + * mark dest files (that excluded with source --exclude-path) also for exclusion + */ + if (current.backup_mode != BACKUP_MODE_FULL) + { + elog(INFO, "Removing redundant files in destination directory"); + parray_qsort(dest_filelist, pgFileCompareRelPathWithExternalDesc); + for (i = 0; i < parray_num(dest_filelist); i++) + { + bool redundant = true; + pgFile *file = (pgFile *) parray_get(dest_filelist, i); + pgFile **src_file = NULL; + + //TODO optimize it and use some merge-like algorithm + //instead of bsearch for each file. + src_file = (pgFile **) parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal); + + if (src_file!= NULL && !(*src_file)->excluded && file->excluded) + (*src_file)->excluded = true; + + if (src_file!= NULL || file->excluded) + redundant = false; + + /* pg_filenode.map are always copied, because it's crc cannot be trusted */ + Assert(file->external_dir_num == 0); + if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) + redundant = true; + /* global/pg_control.pbk.bak is always keeped, because it's needed for restart failed incremental restore */ + if (pg_strcasecmp(file->rel_path, XLOG_CONTROL_BAK_FILE) == 0) + redundant = false; + + /* if file does not exists in destination list, then we can safely unlink it */ + if (redundant) + { + char fullpath[MAXPGPATH]; + + join_path_components(fullpath, dest_pgdata, file->rel_path); + if (!dry_run) + { + fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); + } + elog(LOG, "Deleted file \"%s\"", fullpath); + + /* shrink dest pgdata list */ + pgFileFree(file); + parray_remove(dest_filelist, i); + i--; + } + } + } + + /* clear file locks */ + pfilearray_clear_locks(source_filelist); + + /* Sort by size for load balancing */ + parray_qsort(source_filelist, pgFileCompareSizeDesc); + + /* Sort the array for binary search */ + if (dest_filelist) + parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal); + + join_path_components(dest_pg_control_fullpath, dest_pgdata, XLOG_CONTROL_FILE); + join_path_components(dest_pg_control_bak_fullpath, dest_pgdata, XLOG_CONTROL_BAK_FILE); + /* + * rename (if it exist) dest control file before restoring + * if it doesn't exist, that mean, that we already restoring in a previously failed + * pgdata, where XLOG_CONTROL_BAK_FILE exist + */ + if (current.backup_mode != BACKUP_MODE_FULL && !dry_run) + { + if (!fio_access(dest_pg_control_fullpath, F_OK, FIO_LOCAL_HOST)) + { + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + + if(!fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST)) + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + fio_rename(dest_pg_control_fullpath, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } + } + + /* run copy threads */ + elog(INFO, "Start transferring data files"); + time(&start_time); + transfered_datafiles_bytes = catchup_multithreaded_copy(num_threads, &source_node_info, + source_pgdata, dest_pgdata, + source_filelist, dest_filelist, + dest_redo.lsn, current.backup_mode); + catchup_isok = transfered_datafiles_bytes != -1; + + /* at last copy control file */ + if (catchup_isok && !dry_run) + { + char from_fullpath[MAXPGPATH]; + char to_fullpath[MAXPGPATH]; + join_path_components(from_fullpath, source_pgdata, source_pg_control_file->rel_path); + join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); + copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, + to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); + transfered_datafiles_bytes += source_pg_control_file->size; + + /* Now backup control file can be deled */ + if (current.backup_mode != BACKUP_MODE_FULL && !fio_access(dest_pg_control_bak_fullpath, F_OK, FIO_LOCAL_HOST)){ + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } + } + + if (!catchup_isok && !dry_run) + { + char pretty_time[20]; + char pretty_transfered_data_bytes[20]; + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); + + elog(ERROR, "Catchup failed. Transfered: %s, time elapsed: %s", + pretty_transfered_data_bytes, pretty_time); + } + + /* Notify end of backup */ + { + //REVIEW Is it relevant to catchup? I suppose it isn't, since catchup is a new code. + //If we do need it, please write a comment explaining that. + /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ + int timeout = (instance_config.archive_timeout > 0) ? + instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; + char *stop_backup_query_text = NULL; + + pg_silent_client_messages(source_conn); + + /* Execute pg_stop_backup using PostgreSQL connection */ + pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, exclusive_backup, &stop_backup_query_text); + + /* + * Wait for the result of pg_stop_backup(), but no longer than + * archive_timeout seconds + */ + pg_stop_backup_consume(source_conn, source_node_info.server_version, exclusive_backup, timeout, stop_backup_query_text, &stop_backup_result); + + /* Cleanup */ + pg_free(stop_backup_query_text); + } + + if (!dry_run) + wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); + +#if PG_VERSION_NUM >= 90600 + /* Write backup_label */ + Assert(stop_backup_result.backup_label_content != NULL); + if (!dry_run) + { + pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", + stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, + NULL); + } + free(stop_backup_result.backup_label_content); + stop_backup_result.backup_label_content = NULL; + stop_backup_result.backup_label_content_len = 0; + + /* tablespace_map */ + if (stop_backup_result.tablespace_map_content != NULL) + { + // TODO what if tablespace is created during catchup? + /* Because we have already created symlinks in pg_tblspc earlier, + * we do not need to write the tablespace_map file. + * So this call is unnecessary: + * pg_stop_backup_write_file_helper(dest_pgdata, PG_TABLESPACE_MAP_FILE, "tablespace map", + * stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, + * NULL); + */ + free(stop_backup_result.tablespace_map_content); + stop_backup_result.tablespace_map_content = NULL; + stop_backup_result.tablespace_map_content_len = 0; + } +#endif + + /* wait for end of wal streaming and calculate wal size transfered */ + if (!dry_run) + { + parray *wal_files_list = NULL; + wal_files_list = parray_new(); + + if (wait_WAL_streaming_end(wal_files_list)) + elog(ERROR, "WAL streaming failed"); + + for (i = 0; i < parray_num(wal_files_list); i++) + { + pgFile *file = (pgFile *) parray_get(wal_files_list, i); + transfered_walfiles_bytes += file->size; + } + + parray_walk(wal_files_list, pgFileFree); + parray_free(wal_files_list); + wal_files_list = NULL; + } + + /* + * In case of backup from replica >= 9.6 we must fix minRecPoint + */ + if (current.from_replica && !exclusive_backup) + { + set_min_recovery_point(source_pg_control_file, dest_pgdata, current.stop_lsn); + } + + /* close ssh session in main thread */ + fio_disconnect(); + + /* fancy reporting */ + { + char pretty_transfered_data_bytes[20]; + char pretty_transfered_wal_bytes[20]; + char pretty_time[20]; + + time(&end_time); + pretty_time_interval(difftime(end_time, start_time), + pretty_time, lengthof(pretty_time)); + pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); + pretty_size(transfered_walfiles_bytes, pretty_transfered_wal_bytes, lengthof(pretty_transfered_wal_bytes)); + + elog(INFO, "Databases synchronized. Transfered datafiles size: %s, transfered wal size: %s, time elapsed: %s", + pretty_transfered_data_bytes, pretty_transfered_wal_bytes, pretty_time); + + if (current.backup_mode != BACKUP_MODE_FULL) + elog(INFO, "Catchup incremental ratio (less is better): %.f%% (%s/%s)", + ((float) transfered_datafiles_bytes / current.pgdata_bytes) * 100, + pretty_transfered_data_bytes, pretty_source_bytes); + } + + /* Sync all copied files unless '--no-sync' flag is used */ + if (sync_dest_files && !dry_run) + catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); + else + elog(WARNING, "Files are not synced to disk"); + + /* Cleanup */ + if (dest_filelist && !dry_run) + { + parray_walk(dest_filelist, pgFileFree); + } + parray_free(dest_filelist); + parray_walk(source_filelist, pgFileFree); + parray_free(source_filelist); + pgFileFree(source_pg_control_file); + + return 0; +} diff --git a/src/checkdb.c b/src/checkdb.c index 3a97fc2c7..2a7d4e9eb 100644 --- a/src/checkdb.c +++ b/src/checkdb.c @@ -83,6 +83,7 @@ typedef struct pg_indexEntry char *name; char *namespace; bool heapallindexed_is_supported; + bool checkunique_is_supported; /* schema where amcheck extension is located */ char *amcheck_nspname; /* lock for synchronization of parallel threads */ @@ -144,7 +145,7 @@ check_files(void *arg) /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "interrupted during checkdb"); + elog(ERROR, "Interrupted during checkdb"); /* No need to check directories */ if (S_ISDIR(file->mode)) @@ -292,6 +293,7 @@ check_indexes(void *arg) int i; check_indexes_arg *arguments = (check_indexes_arg *) arg; int n_indexes = 0; + my_thread_num = arguments->thread_num; if (arguments->index_list) n_indexes = parray_num(arguments->index_list); @@ -350,16 +352,20 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, { PGresult *res; char *amcheck_nspname = NULL; + char *amcheck_extname = NULL; + char *amcheck_extversion = NULL; int i; bool heapallindexed_is_supported = false; + bool checkunique_is_supported = false; parray *index_list = NULL; + /* Check amcheck extension version */ res = pgut_execute(db_conn, "SELECT " "extname, nspname, extversion " - "FROM pg_namespace n " - "JOIN pg_extension e " + "FROM pg_catalog.pg_namespace n " + "JOIN pg_catalog.pg_extension e " "ON n.oid=e.extnamespace " - "WHERE e.extname IN ('amcheck', 'amcheck_next') " + "WHERE e.extname IN ('amcheck'::name, 'amcheck_next'::name) " "ORDER BY extversion DESC " "LIMIT 1", 0, NULL); @@ -378,24 +384,68 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, return NULL; } + amcheck_extname = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); + strcpy(amcheck_extname, PQgetvalue(res, 0, 0)); amcheck_nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); strcpy(amcheck_nspname, PQgetvalue(res, 0, 1)); + amcheck_extversion = pgut_malloc(strlen(PQgetvalue(res, 0, 2)) + 1); + strcpy(amcheck_extversion, PQgetvalue(res, 0, 2)); + PQclear(res); /* heapallindexed_is_supported is database specific */ - if (strcmp(PQgetvalue(res, 0, 2), "1.0") != 0 && - strcmp(PQgetvalue(res, 0, 2), "1") != 0) + /* TODO this is wrong check, heapallindexed supported also in 1.1.1, 1.2 and 1.2.1... */ + if (strcmp(amcheck_extversion, "1.0") != 0 && + strcmp(amcheck_extversion, "1") != 0) heapallindexed_is_supported = true; elog(INFO, "Amchecking database '%s' using extension '%s' " "version %s from schema '%s'", - dbname, PQgetvalue(res, 0, 0), - PQgetvalue(res, 0, 2), PQgetvalue(res, 0, 1)); + dbname, amcheck_extname, + amcheck_extversion, amcheck_nspname); if (!heapallindexed_is_supported && heapallindexed) elog(WARNING, "Extension '%s' version %s in schema '%s'" "do not support 'heapallindexed' option", - PQgetvalue(res, 0, 0), PQgetvalue(res, 0, 2), - PQgetvalue(res, 0, 1)); + amcheck_extname, amcheck_extversion, + amcheck_nspname); + +#ifndef PGPRO_EE + /* + * Will support when the vanilla patch will commited https://commitfest.postgresql.org/32/2976/ + */ + checkunique_is_supported = false; +#else + /* + * Check bt_index_check function signature to determine support of checkunique parameter + * This can't be exactly checked by checking extension version, + * For example, 1.1.1 and 1.2.1 supports this parameter, but 1.2 doesn't (PGPROEE-12.4.1) + */ + res = pgut_execute(db_conn, "SELECT " + " oid " + "FROM pg_catalog.pg_proc " + "WHERE " + " pronamespace = $1::regnamespace " + "AND proname = 'bt_index_check' " + "AND 'checkunique' = ANY(proargnames) " + "AND (pg_catalog.string_to_array(proargtypes::text, ' ')::regtype[])[pg_catalog.array_position(proargnames, 'checkunique')] = 'bool'::regtype", + 1, (const char **) &amcheck_nspname); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + elog(ERROR, "Cannot check 'checkunique' option is supported in bt_index_check function %s: %s", + dbname, PQerrorMessage(db_conn)); + } + + checkunique_is_supported = PQntuples(res) >= 1; + PQclear(res); +#endif + + if (!checkunique_is_supported && checkunique) + elog(WARNING, "Extension '%s' version %s in schema '%s' " + "do not support 'checkunique' parameter", + amcheck_extname, amcheck_extversion, + amcheck_nspname); /* * In order to avoid duplicates, select global indexes @@ -411,7 +461,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " "ORDER BY nmspc.nspname DESC", 0, NULL); } @@ -423,8 +475,10 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' AND cls.relpersistence != 't' AND " - "(cls.reltablespace IN " + "WHERE am.amname='btree' " + "AND cls.relpersistence != 't' " + "AND cls.relkind != 'I' " + "AND (cls.reltablespace IN " "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " "OR cls.reltablespace = 0) " "ORDER BY nmspc.nspname DESC", @@ -439,7 +493,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, char *namespace = NULL; /* index oid */ - ind->indexrelid = atoi(PQgetvalue(res, i, 0)); + ind->indexrelid = atoll(PQgetvalue(res, i, 0)); /* index relname */ name = PQgetvalue(res, i, 1); @@ -452,6 +506,7 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, strcpy(ind->namespace, namespace); /* enough buffer size guaranteed */ ind->heapallindexed_is_supported = heapallindexed_is_supported; + ind->checkunique_is_supported = checkunique_is_supported; ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); strcpy(ind->amcheck_nspname, amcheck_nspname); pg_atomic_clear_flag(&ind->lock); @@ -463,6 +518,9 @@ get_index_list(const char *dbname, bool first_db_with_amcheck, } PQclear(res); + free(amcheck_extversion); + free(amcheck_nspname); + free(amcheck_extname); return index_list; } @@ -472,38 +530,46 @@ static bool amcheck_one_index(check_indexes_arg *arguments, pg_indexEntry *ind) { - PGresult *res; - char *params[2]; + PGresult *res; + char *params[3]; + static const char *queries[] = { + "SELECT %s.bt_index_check(index => $1)", + "SELECT %s.bt_index_check(index => $1, heapallindexed => $2)", + "SELECT %s.bt_index_check(index => $1, heapallindexed => $2, checkunique => $3)", + }; + int params_count; char *query = NULL; - params[0] = palloc(64); + if (interrupted) + elog(ERROR, "Interrupted"); +#define INDEXRELID 0 +#define HEAPALLINDEXED 1 +#define CHECKUNIQUE 2 /* first argument is index oid */ - sprintf(params[0], "%i", ind->indexrelid); + params[INDEXRELID] = palloc(64); + sprintf(params[INDEXRELID], "%u", ind->indexrelid); /* second argument is heapallindexed */ - params[1] = heapallindexed ? "true" : "false"; + params[HEAPALLINDEXED] = heapallindexed ? "true" : "false"; + /* third optional argument is checkunique */ + params[CHECKUNIQUE] = checkunique ? "true" : "false"; +#undef CHECKUNIQUE +#undef HEAPALLINDEXED - if (interrupted) - elog(ERROR, "Interrupted"); - - if (ind->heapallindexed_is_supported) - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1, $2)")+1); - sprintf(query, "SELECT %s.bt_index_check($1, $2)", ind->amcheck_nspname); + params_count = ind->checkunique_is_supported ? + 3 : + ( ind->heapallindexed_is_supported ? 2 : 1 ); - res = pgut_execute_parallel(arguments->conn_arg.conn, - arguments->conn_arg.cancel_conn, - query, 2, (const char **)params, true, true, true); - } - else - { - query = palloc(strlen(ind->amcheck_nspname)+strlen("SELECT .bt_index_check($1)")+1); - sprintf(query, "SELECT %s.bt_index_check($1)", ind->amcheck_nspname); + /* + * Prepare query text with schema name + * +1 for \0 and -2 for %s + */ + query = palloc(strlen(ind->amcheck_nspname) + strlen(queries[params_count - 1]) + 1 - 2); + sprintf(query, queries[params_count - 1], ind->amcheck_nspname); - res = pgut_execute_parallel(arguments->conn_arg.conn, + res = pgut_execute_parallel(arguments->conn_arg.conn, arguments->conn_arg.cancel_conn, - query, 1, (const char **)params, true, true, true); - } + query, params_count, (const char **)params, true, true, true); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -511,7 +577,7 @@ amcheck_one_index(check_indexes_arg *arguments, arguments->thread_num, arguments->conn_opt.pgdatabase, ind->namespace, ind->name, PQresultErrorMessage(res)); - pfree(params[0]); + pfree(params[INDEXRELID]); pfree(query); PQclear(res); return false; @@ -521,7 +587,8 @@ amcheck_one_index(check_indexes_arg *arguments, arguments->thread_num, arguments->conn_opt.pgdatabase, ind->namespace, ind->name); - pfree(params[0]); + pfree(params[INDEXRELID]); +#undef INDEXRELID pfree(query); PQclear(res); return true; @@ -555,8 +622,8 @@ do_amcheck(ConnectionOptions conn_opt, PGconn *conn) res_db = pgut_execute(conn, "SELECT datname, oid, dattablespace " - "FROM pg_database " - "WHERE datname NOT IN ('template0', 'template1')", + "FROM pg_catalog.pg_database " + "WHERE datname NOT IN ('template0'::name, 'template1'::name)", 0, NULL); /* we don't need this connection anymore */ @@ -683,7 +750,7 @@ do_checkdb(bool need_amcheck, if (!skip_block_validation) { if (!pgdata) - elog(ERROR, "required parameter not specified: PGDATA " + elog(ERROR, "Required parameter not specified: PGDATA " "(-D, --pgdata)"); /* get node info */ diff --git a/src/configure.c b/src/configure.c index 1aae3df13..964548343 100644 --- a/src/configure.c +++ b/src/configure.c @@ -17,10 +17,14 @@ static void assign_log_level_console(ConfigOption *opt, const char *arg); static void assign_log_level_file(ConfigOption *opt, const char *arg); +static void assign_log_format_console(ConfigOption *opt, const char *arg); +static void assign_log_format_file(ConfigOption *opt, const char *arg); static void assign_compress_alg(ConfigOption *opt, const char *arg); static char *get_log_level_console(ConfigOption *opt); static char *get_log_level_file(ConfigOption *opt); +static char *get_log_format_console(ConfigOption *opt); +static char *get_log_format_file(ConfigOption *opt); static char *get_compress_alg(ConfigOption *opt); static void show_configure_start(void); @@ -49,7 +53,7 @@ ConfigOption instance_options[] = /* Instance options */ { 's', 'D', "pgdata", - &instance_config.pgdata, SOURCE_CMD, 0, + &instance_config.pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -66,49 +70,49 @@ ConfigOption instance_options[] = #endif { 's', 'E', "external-dirs", - &instance_config.external_dir_str, SOURCE_CMD, 0, + &instance_config.external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance_config.conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance_config.conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance_config.conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance_config.conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance_config.master_conn_opt.pghost, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance_config.master_conn_opt.pgport, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance_config.master_conn_opt.pguser, SOURCE_CMD, 0, + &instance_config.master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -124,17 +128,17 @@ ConfigOption instance_options[] = }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { @@ -145,100 +149,110 @@ ConfigOption instance_options[] = /* Logging options */ { 'f', 212, "log-level-console", - assign_log_level_console, SOURCE_CMD, 0, + assign_log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_console }, { 'f', 213, "log-level-file", - assign_log_level_file, SOURCE_CMD, 0, + assign_log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, get_log_level_file }, { - 's', 214, "log-filename", - &instance_config.logger.log_filename, SOURCE_CMD, 0, + 'f', 214, "log-format-console", + assign_log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, + OPTION_LOG_GROUP, 0, get_log_format_console + }, + { + 'f', 215, "log-format-file", + assign_log_format_file, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, 0, get_log_format_file + }, + { + 's', 216, "log-filename", + &instance_config.logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 215, "error-log-filename", - &instance_config.logger.error_log_filename, SOURCE_CMD, 0, + 's', 217, "error-log-filename", + &instance_config.logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 216, "log-directory", - &instance_config.logger.log_directory, SOURCE_CMD, 0, + 's', 218, "log-directory", + &instance_config.logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 217, "log-rotation-size", + 'U', 219, "log-rotation-size", &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 218, "log-rotation-age", + 'U', 220, "log-rotation-age", &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 219, "retention-redundancy", - &instance_config.retention_redundancy, SOURCE_CMD, 0, + 'u', 221, "retention-redundancy", + &instance_config.retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 220, "retention-window", - &instance_config.retention_window, SOURCE_CMD, 0, + 'u', 222, "retention-window", + &instance_config.retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 221, "wal-depth", - &instance_config.wal_depth, SOURCE_CMD, 0, + 'u', 223, "wal-depth", + &instance_config.wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 'f', 222, "compress-algorithm", - assign_compress_alg, SOURCE_CMD, 0, + 'f', 224, "compress-algorithm", + assign_compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, get_compress_alg }, { - 'u', 223, "compress-level", - &instance_config.compress_level, SOURCE_CMD, 0, + 'u', 225, "compress-level", + &instance_config.compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 224, "remote-proto", - &instance_config.remote.proto, SOURCE_CMD, 0, + 's', 226, "remote-proto", + &instance_config.remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-host", - &instance_config.remote.host, SOURCE_CMD, 0, + 's', 227, "remote-host", + &instance_config.remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-port", - &instance_config.remote.port, SOURCE_CMD, 0, + 's', 228, "remote-port", + &instance_config.remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-path", - &instance_config.remote.path, SOURCE_CMD, 0, + 's', 229, "remote-path", + &instance_config.remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "remote-user", - &instance_config.remote.user, SOURCE_CMD, 0, + 's', 230, "remote-user", + &instance_config.remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-options", - &instance_config.remote.ssh_options, SOURCE_CMD, 0, + 's', 231, "ssh-options", + &instance_config.remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 230, "ssh-config", - &instance_config.remote.ssh_config, SOURCE_CMD, 0, + 's', 232, "ssh-config", + &instance_config.remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } @@ -255,7 +269,7 @@ static const char *current_group = NULL; * Show configure options including default values. */ void -do_show_config(void) +do_show_config(bool show_base_units) { int i; @@ -263,10 +277,13 @@ do_show_config(void) for (i = 0; instance_options[i].type; i++) { + if (show_base_units && strchr("bBiIuU", instance_options[i].type) && instance_options[i].get_value == *option_get_value) + instance_options[i].flags |= GET_VAL_IN_BASE_UNITS; /* Set flag */ if (show_format == SHOW_PLAIN) show_configure_plain(&instance_options[i]); else show_configure_json(&instance_options[i]); + instance_options[i].flags &= ~(GET_VAL_IN_BASE_UNITS); /* Reset flag. It was resetted in option_get_value(). Probably this reset isn't needed */ } show_configure_end(); @@ -277,18 +294,16 @@ do_show_config(void) * values into the file. */ void -do_set_config(bool missing_ok) +do_set_config(InstanceState *instanceState, bool missing_ok) { - char path[MAXPGPATH]; char path_temp[MAXPGPATH]; FILE *fp; int i; - join_path_components(path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); + snprintf(path_temp, sizeof(path_temp), "%s.tmp", instanceState->instance_config_path); - if (!missing_ok && !fileExists(path, FIO_LOCAL_HOST)) - elog(ERROR, "Configuration file \"%s\" doesn't exist", path); + if (!missing_ok && !fileExists(instanceState->instance_config_path, FIO_LOCAL_HOST)) + elog(ERROR, "Configuration file \"%s\" doesn't exist", instanceState->instance_config_path); fp = fopen(path_temp, "wt"); if (fp == NULL) @@ -299,6 +314,7 @@ do_set_config(bool missing_ok) for (i = 0; instance_options[i].type; i++) { + int rc = 0; ConfigOption *opt = &instance_options[i]; char *value; @@ -319,20 +335,32 @@ do_set_config(bool missing_ok) } if (strchr(value, ' ')) - fprintf(fp, "%s = '%s'\n", opt->lname, value); + rc = fprintf(fp, "%s = '%s'\n", opt->lname, value); else - fprintf(fp, "%s = %s\n", opt->lname, value); + rc = fprintf(fp, "%s = %s\n", opt->lname, value); + + if (rc < 0) + elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); + pfree(value); } - fclose(fp); + if (ferror(fp) || fflush(fp)) + elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); + + if (fclose(fp)) + elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp); + + if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0) + elog(ERROR, "Failed to sync temp configuration file \"%s\": %s", + path_temp, strerror(errno)); - if (rename(path_temp, path) < 0) + if (rename(path_temp, instanceState->instance_config_path) < 0) { int errno_temp = errno; unlink(path_temp); elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno_temp)); + path_temp, instanceState->instance_config_path, strerror(errno_temp)); } } @@ -341,8 +369,6 @@ init_config(InstanceConfig *config, const char *instance_name) { MemSet(config, 0, sizeof(InstanceConfig)); - config->name = pgut_strdup(instance_name); - /* * Starting from PostgreSQL 11 WAL segment size may vary. Prior to * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. @@ -374,12 +400,13 @@ init_config(InstanceConfig *config, const char *instance_name) * read instance config from file */ InstanceConfig * -readInstanceConfigFile(const char *instance_name) +readInstanceConfigFile(InstanceState *instanceState) { - char path[MAXPGPATH]; InstanceConfig *instance = pgut_new(InstanceConfig); char *log_level_console = NULL; char *log_level_file = NULL; + char *log_format_console = NULL; + char *log_format_file = NULL; char *compress_alg = NULL; int parsed_options; @@ -388,7 +415,7 @@ readInstanceConfigFile(const char *instance_name) /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, { @@ -405,49 +432,49 @@ readInstanceConfigFile(const char *instance_name) #endif { 's', 'E', "external-dirs", - &instance->external_dir_str, SOURCE_CMD, 0, + &instance->external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Connection options */ { 's', 'd', "pgdatabase", - &instance->conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'h', "pghost", - &instance->conn_opt.pghost, SOURCE_CMD, 0, + &instance->conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'p', "pgport", - &instance->conn_opt.pgport, SOURCE_CMD, 0, + &instance->conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, { 's', 'U', "pguser", - &instance->conn_opt.pguser, SOURCE_CMD, 0, + &instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_CONN_GROUP, 0, option_get_value }, /* Replica options */ { 's', 202, "master-db", - &instance->master_conn_opt.pgdatabase, SOURCE_CMD, 0, + &instance->master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 203, "master-host", - &instance->master_conn_opt.pghost, SOURCE_CMD, 0, + &instance->master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 204, "master-port", - &instance->master_conn_opt.pgport, SOURCE_CMD, 0, + &instance->master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { 's', 205, "master-user", - &instance->master_conn_opt.pguser, SOURCE_CMD, 0, + &instance->master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REPLICA_GROUP, 0, option_get_value }, { @@ -463,160 +490,160 @@ readInstanceConfigFile(const char *instance_name) }, { 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, 0, + &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, 0, + &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, 0, + &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, { 's', 211, "restore-command", - &instance->restore_command, SOURCE_CMD, 0, + &instance->restore_command, SOURCE_CMD, SOURCE_DEFAULT, OPTION_ARCHIVE_GROUP, 0, option_get_value }, /* Instance options */ { 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, 0, + &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, OPTION_INSTANCE_GROUP, 0, option_get_value }, /* Logging options */ { 's', 212, "log-level-console", - &log_level_console, SOURCE_CMD, 0, + &log_level_console, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { 's', 213, "log-level-file", - &log_level_file, SOURCE_CMD, 0, + &log_level_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 214, "log-filename", - &instance->logger.log_filename, SOURCE_CMD, 0, + 's', 214, "log-format-console", + &log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 215, "error-log-filename", - &instance->logger.error_log_filename, SOURCE_CMD, 0, + 's', 215, "log-format-file", + &log_format_file, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 's', 216, "log-directory", - &instance->logger.log_directory, SOURCE_CMD, 0, + 's', 216, "log-filename", + &instance->logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 'U', 217, "log-rotation-size", + 's', 217, "error-log-filename", + &instance->logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 's', 218, "log-directory", + &instance->logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, + OPTION_LOG_GROUP, 0, option_get_value + }, + { + 'U', 219, "log-rotation-size", &instance->logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value }, { - 'U', 218, "log-rotation-age", + 'U', 220, "log-rotation-age", &instance->logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value }, /* Retention options */ { - 'u', 219, "retention-redundancy", - &instance->retention_redundancy, SOURCE_CMD, 0, + 'u', 221, "retention-redundancy", + &instance->retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 220, "retention-window", - &instance->retention_window, SOURCE_CMD, 0, + 'u', 222, "retention-window", + &instance->retention_window, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, { - 'u', 221, "wal-depth", - &instance->wal_depth, SOURCE_CMD, 0, + 'u', 223, "wal-depth", + &instance->wal_depth, SOURCE_CMD, SOURCE_DEFAULT, OPTION_RETENTION_GROUP, 0, option_get_value }, /* Compression options */ { - 's', 222, "compress-algorithm", - &compress_alg, SOURCE_CMD, 0, + 's', 224, "compress-algorithm", + &compress_alg, SOURCE_CMD, SOURCE_DEFAULT, OPTION_LOG_GROUP, 0, option_get_value }, { - 'u', 223, "compress-level", - &instance->compress_level, SOURCE_CMD, 0, + 'u', 225, "compress-level", + &instance->compress_level, SOURCE_CMD, SOURCE_DEFAULT, OPTION_COMPRESS_GROUP, 0, option_get_value }, /* Remote backup options */ { - 's', 224, "remote-proto", - &instance->remote.proto, SOURCE_CMD, 0, + 's', 226, "remote-proto", + &instance->remote.proto, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 225, "remote-host", - &instance->remote.host, SOURCE_CMD, 0, + 's', 227, "remote-host", + &instance->remote.host, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 226, "remote-port", - &instance->remote.port, SOURCE_CMD, 0, + 's', 228, "remote-port", + &instance->remote.port, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 227, "remote-path", - &instance->remote.path, SOURCE_CMD, 0, + 's', 229, "remote-path", + &instance->remote.path, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 228, "remote-user", - &instance->remote.user, SOURCE_CMD, 0, + 's', 230, "remote-user", + &instance->remote.user, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 229, "ssh-options", - &instance->remote.ssh_options, SOURCE_CMD, 0, + 's', 231, "ssh-options", + &instance->remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { - 's', 230, "ssh-config", - &instance->remote.ssh_config, SOURCE_CMD, 0, + 's', 232, "ssh-config", + &instance->remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, OPTION_REMOTE_GROUP, 0, option_get_value }, { 0 } }; - init_config(instance, instance_name); + init_config(instance, instanceState->instance_name); - sprintf(instance->backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - canonicalize_path(instance->backup_instance_path); - - sprintf(instance->arclog_path, "%s/%s/%s", - backup_path, "wal", instance_name); - canonicalize_path(instance->arclog_path); - - join_path_components(path, instance->backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - - if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) { - elog(WARNING, "Control file \"%s\" doesn't exist", path); + elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); pfree(instance); return NULL; } - parsed_options = config_read_opt(path, instance_options, WARNING, true, true); + parsed_options = config_read_opt(instanceState->instance_config_path, + instance_options, WARNING, true, true); if (parsed_options == 0) { - elog(WARNING, "Control file \"%s\" is empty", path); + elog(WARNING, "Control file \"%s\" is empty", instanceState->instance_config_path); pfree(instance); return NULL; } @@ -627,6 +654,12 @@ readInstanceConfigFile(const char *instance_name) if (log_level_file) instance->logger.log_level_file = parse_log_level(log_level_file); + if (log_format_console) + instance->logger.log_format_console = parse_log_format(log_format_console); + + if (log_format_file) + instance->logger.log_format_file = parse_log_format(log_format_file); + if (compress_alg) instance->compress_alg = parse_compress_alg(compress_alg); @@ -637,7 +670,6 @@ readInstanceConfigFile(const char *instance_name) #endif return instance; - } static void @@ -652,6 +684,18 @@ assign_log_level_file(ConfigOption *opt, const char *arg) instance_config.logger.log_level_file = parse_log_level(arg); } +static void +assign_log_format_console(ConfigOption *opt, const char *arg) +{ + instance_config.logger.log_format_console = parse_log_format(arg); +} + +static void +assign_log_format_file(ConfigOption *opt, const char *arg) +{ + instance_config.logger.log_format_file = parse_log_format(arg); +} + static void assign_compress_alg(ConfigOption *opt, const char *arg) { @@ -670,6 +714,18 @@ get_log_level_file(ConfigOption *opt) return pstrdup(deparse_log_level(instance_config.logger.log_level_file)); } +static char * +get_log_format_console(ConfigOption *opt) +{ + return pstrdup(deparse_log_format(instance_config.logger.log_format_console)); +} + +static char * +get_log_format_file(ConfigOption *opt) +{ + return pstrdup(deparse_log_format(instance_config.logger.log_format_file)); +} + static char * get_compress_alg(ConfigOption *opt) { @@ -748,6 +804,6 @@ show_configure_json(ConfigOption *opt) return; json_add_value(&show_buf, opt->lname, value, json_level, - true); + !(opt->flags & GET_VAL_IN_BASE_UNITS)); pfree(value); } diff --git a/src/data.c b/src/data.c index 6fc904ca0..1a9616bae 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -28,10 +28,10 @@ typedef struct DataPage { BackupPageHeader bph; - char data[BLCKSZ]; + char data[BLCKSZ]; } DataPage; -static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, +static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, pg_crc32 *crc, bool use_crc32c); #ifdef HAVE_LIBZ @@ -40,9 +40,9 @@ static int32 zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, int level) { - uLongf compressed_size = dst_size; - int rc = compress2(dst, &compressed_size, src, src_size, - level); + uLongf compressed_size = dst_size; + int rc = compress2(dst, &compressed_size, src, src_size, + level); return rc == Z_OK ? compressed_size : rc; } @@ -51,8 +51,8 @@ zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, static int32 zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) { - uLongf dest_len = dst_size; - int rc = uncompress(dst, &dest_len, src, src_size); + uLongf dest_len = dst_size; + int rc = uncompress(dst, &dest_len, src, src_size); return rc == Z_OK ? dest_len : rc; } @@ -63,7 +63,7 @@ zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) * written in the destination buffer, or -1 if compression fails. */ int32 -do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, +do_compress(void *dst, size_t dst_size, void const *src, size_t src_size, CompressAlg alg, int level, const char **errormsg) { switch (alg) @@ -73,13 +73,13 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_compress(dst, dst_size, src, src_size, level); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } + { + int32 ret; + ret = zlib_compress(dst, dst_size, src, src_size, level); + if (ret < Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: return pglz_compress(src, src_size, dst, PGLZ_strategy_always); @@ -93,25 +93,25 @@ do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, * decompressed in the destination buffer, or -1 if decompression fails. */ int32 -do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, +do_decompress(void *dst, size_t dst_size, void const *src, size_t src_size, CompressAlg alg, const char **errormsg) { switch (alg) { case NONE_COMPRESS: case NOT_DEFINED_COMPRESS: - if (errormsg) + if (errormsg) *errormsg = "Invalid compression algorithm"; return -1; #ifdef HAVE_LIBZ case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_decompress(dst, dst_size, src, src_size); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } + { + int32 ret; + ret = zlib_decompress(dst, dst_size, src, src_size); + if (ret < Z_OK && errormsg) + *errormsg = zError(ret); + return ret; + } #endif case PGLZ_COMPRESS: @@ -125,7 +125,6 @@ do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, return -1; } - #define ZLIB_MAGIC 0x78 /* @@ -143,7 +142,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) phdr = (PageHeader) page; /* First check if page header is valid (it seems to be fast enough check) */ - if (!(PageGetPageSize(phdr) == BLCKSZ && + if (!(PageGetPageSize(page) == BLCKSZ && // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && @@ -162,7 +161,7 @@ page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) /* For zlib we can check page magic: * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like */ - if (alg == ZLIB_COMPRESS && *(char*)page != ZLIB_MAGIC) + if (alg == ZLIB_COMPRESS && *(char *)page != ZLIB_MAGIC) { return false; } @@ -182,7 +181,7 @@ parse_page(Page page, XLogRecPtr *lsn) /* Get lsn from page header */ *lsn = PageXLogRecPtrGet(phdr->pd_lsn); - if (PageGetPageSize(phdr) == BLCKSZ && + if (PageGetPageSize(page) == BLCKSZ && // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && phdr->pd_lower >= SizeOfPageHeaderData && @@ -204,10 +203,10 @@ get_header_errormsg(Page page, char **errormsg) PageHeader phdr = (PageHeader) page; *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - if (PageGetPageSize(phdr) != BLCKSZ) + if (PageGetPageSize(page) != BLCKSZ) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " "page size %lu is not equal to block size %u", - PageGetPageSize(phdr), BLCKSZ); + PageGetPageSize(page), BLCKSZ); else if (phdr->pd_lower < SizeOfPageHeaderData) snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " @@ -268,7 +267,7 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) * PageIsOk(0) if page was successfully retrieved * PageIsTruncated(-1) if the page was truncated * SkipCurrentPage(-2) if we need to skip this page, - * only used for DELTA backup + * only used for DELTA and PTRACK backup * PageIsCorrupted(-3) if the page checksum mismatch * or header corruption, * only used for checkdb @@ -276,20 +275,18 @@ get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) * return it to the caller */ static int32 -prepare_page(ConnectionArgs *conn_arg, - pgFile *file, XLogRecPtr prev_backup_start_lsn, +prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, BlockNumber blknum, FILE *in, BackupMode backup_mode, Page page, bool strict, uint32 checksum_version, - int ptrack_version_num, - const char *ptrack_schema, const char *from_fullpath, PageState *page_st) { int try_again = PAGE_READ_ATTEMPTS; bool page_is_valid = false; BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; + int rc = 0; /* check for interrupt */ if (interrupted || thread_interrupted) @@ -300,171 +297,113 @@ prepare_page(ConnectionArgs *conn_arg, * Under high write load it's possible that we've read partly * flushed page, so try several times before throwing an error. */ - if (backup_mode != BACKUP_MODE_DIFF_PTRACK || ptrack_version_num >= 20) + while (!page_is_valid && try_again--) { - int rc = 0; - while (!page_is_valid && try_again--) - { - /* read the block */ - int read_len = fio_pread(in, page, blknum * BLCKSZ); + /* read the block */ + int read_len = fio_pread(in, page, blknum * BLCKSZ); - /* The block could have been truncated. It is fine. */ - if (read_len == 0) - { - elog(VERBOSE, "Cannot read block %u of \"%s\": " - "block truncated", blknum, from_fullpath); - return PageIsTruncated; - } - else if (read_len < 0) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - else if (read_len != BLCKSZ) - elog(WARNING, "Cannot read block %u of \"%s\": " - "read %i of %d, try again", - blknum, from_fullpath, read_len, BLCKSZ); - else + /* The block could have been truncated. It is fine. */ + if (read_len == 0) + { + elog(VERBOSE, "Cannot read block %u of \"%s\": " + "block truncated", blknum, from_fullpath); + return PageIsTruncated; + } + else if (read_len < 0) + elog(ERROR, "Cannot read block %u of \"%s\": %s", + blknum, from_fullpath, strerror(errno)); + else if (read_len != BLCKSZ) + elog(WARNING, "Cannot read block %u of \"%s\": " + "read %i of %d, try again", + blknum, from_fullpath, read_len, BLCKSZ); + else + { + /* We have BLCKSZ of raw data, validate it */ + rc = validate_one_page(page, absolute_blknum, + InvalidXLogRecPtr, page_st, + checksum_version); + switch (rc) { - /* We have BLCKSZ of raw data, validate it */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - switch (rc) - { - case PAGE_IS_ZEROED: - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + case PAGE_IS_ZEROED: + elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); + return PageIsOk; + + case PAGE_IS_VALID: + /* in DELTA or PTRACK modes we must compare lsn */ + if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) + page_is_valid = true; + else return PageIsOk; - - case PAGE_IS_VALID: - /* in DELTA mode we must compare lsn */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA) - page_is_valid = true; - else - return PageIsOk; - break; - - case PAGE_HEADER_IS_INVALID: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", - from_fullpath, blknum); - break; - - case PAGE_CHECKSUM_MISMATCH: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", - from_fullpath, blknum); - break; - default: - Assert(false); - } + break; + + case PAGE_HEADER_IS_INVALID: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", + from_fullpath, blknum); + break; + + case PAGE_CHECKSUM_MISMATCH: + elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", + from_fullpath, blknum); + break; + default: + Assert(false); } } - - /* - * If page is not valid after 100 attempts to read it - * throw an error. - */ - if (!page_is_valid) - { - int elevel = ERROR; - char *errormsg = NULL; - - /* Get the details of corruption */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(page, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(page, &errormsg, - file->segno * RELSEG_SIZE + blknum); - - /* Error out in case of merge or backup without ptrack support; - * issue warning in case of checkdb or backup with ptrack support - */ - if (!strict) - elevel = WARNING; - - if (errormsg) - elog(elevel, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - else - elog(elevel, "Corruption detected in file \"%s\", block %u", - from_fullpath, blknum); - - pg_free(errormsg); - return PageIsCorrupted; - } - - /* Checkdb not going futher */ - if (!strict) - return PageIsOk; + /* avoid re-reading once buffered data, flushing on further attempts, see PBCKP-150 */ + fflush(in); } /* - * Get page via ptrack interface from PostgreSQL shared buffer. - * We do this only in the cases of PTRACK 1.x versions backup + * If page is not valid after PAGE_READ_ATTEMPTS attempts to read it + * throw an error. */ - if (backup_mode == BACKUP_MODE_DIFF_PTRACK - && (ptrack_version_num >= 15 && ptrack_version_num < 20)) - { - int rc = 0; - size_t page_size = 0; - Page ptrack_page = NULL; - ptrack_page = (Page) pg_ptrack_get_block(conn_arg, file->dbOid, file->tblspcOid, - file->relOid, absolute_blknum, &page_size, - ptrack_version_num, ptrack_schema); - - if (ptrack_page == NULL) - /* This block was truncated.*/ - return PageIsTruncated; - - if (page_size != BLCKSZ) - elog(ERROR, "File: \"%s\", block %u, expected block size %d, but read %zu", - from_fullpath, blknum, BLCKSZ, page_size); - - /* - * We need to copy the page that was successfully - * retrieved from ptrack into our output "page" parameter. - */ - memcpy(page, ptrack_page, BLCKSZ); - pg_free(ptrack_page); - - /* - * UPD: It apprears that is possible to get zeroed page or page with invalid header - * from shared buffer. - * Note, that getting page with wrong checksumm from shared buffer is - * acceptable. - */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - - /* It is ok to get zeroed page */ - if (rc == PAGE_IS_ZEROED) - return PageIsOk; + if (!page_is_valid) + { + int elevel = ERROR; + char *errormsg = NULL; - /* Getting page with invalid header from shared buffers is unacceptable */ + /* Get the details of corruption */ if (rc == PAGE_HEADER_IS_INVALID) - { - char *errormsg = NULL; get_header_errormsg(page, &errormsg); - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - } + else if (rc == PAGE_CHECKSUM_MISMATCH) + get_checksum_errormsg(page, &errormsg, + file->segno * RELSEG_SIZE + blknum); - /* - * We must set checksum here, because it is outdated - * in the block recieved from shared buffers. + /* Error out in case of merge or backup without ptrack support; + * issue warning in case of checkdb or backup with ptrack support */ - if (checksum_version) - page_st->checksum = ((PageHeader) page)->pd_checksum = pg_checksum_page(page, absolute_blknum); + if (!strict) + elevel = WARNING; + + if (errormsg) + elog(elevel, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, blknum, errormsg); + else + elog(elevel, "Corruption detected in file \"%s\", block %u", + from_fullpath, blknum); + + pg_free(errormsg); + return PageIsCorrupted; } + /* Checkdb not going futher */ + if (!strict) + return PageIsOk; + /* * Skip page if page lsn is less than START_LSN of parent backup. * Nullified pages must be copied by DELTA backup, just to be safe. */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA && + if ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev && page_st->lsn > 0 && page_st->lsn < prev_backup_start_lsn) { - elog(VERBOSE, "Skipping blknum %u in file: \"%s\"", blknum, from_fullpath); + elog(VERBOSE, "Skipping blknum %u in file: \"%s\", file->exists_in_prev: %s, page_st->lsn: %X/%X, prev_backup_start_lsn: %X/%X", + blknum, from_fullpath, + file->exists_in_prev ? "true" : "false", + (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, + (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn); return SkipCurrentPage; } @@ -481,7 +420,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, { int compressed_size = 0; size_t write_buffer_size = 0; - char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ + char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ BackupPageHeader* bph = (BackupPageHeader*)write_buffer; const char *errormsg = NULL; @@ -522,6 +461,20 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, return compressed_size; } +/* Write page as-is. TODO: make it fastpath option in compress_and_backup_page() */ +static int +write_page(pgFile *file, FILE *out, Page page) +{ + /* write data page */ + if (fio_fwrite(out, page, BLCKSZ) != BLCKSZ) + return -1; + + file->write_size += BLCKSZ; + file->uncompressed_size += BLCKSZ; + + return BLCKSZ; +} + /* * Backup data file in the from_root directory to the to_root directory with * same relative path. If prev_backup_start_lsn is not NULL, only pages with @@ -531,17 +484,15 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, * backup with special header. */ void -backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, +backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, HeaderMap *hdr_map, bool is_merge) { int rc; bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; + char *errmsg = NULL; + BlockNumber err_blknum = 0; /* page headers */ BackupPageHeader2 *headers = NULL; @@ -590,7 +541,7 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, * Such files should be fully copied. */ - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || file->pagemap_isabsent || !file->exists_in_prev || !file->pagemap.bitmap) use_pagemap = false; @@ -600,28 +551,25 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, /* Remote mode */ if (fio_is_remote(FIO_DB_HOST)) { - rc = fio_send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, calg, clevel, checksum_version, /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, + use_pagemap, /* variables for error reporting */ &err_blknum, &errmsg, &headers); } else { /* TODO: stop handling errors internally */ - rc = send_pages(conn_arg, to_fullpath, from_fullpath, file, + rc = send_pages(to_fullpath, from_fullpath, file, /* send prev backup START_LSN */ - backup_mode == BACKUP_MODE_DIFF_DELTA && + (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap ? &file->pagemap : NULL, - &headers, backup_mode, ptrack_version_num, ptrack_schema); + calg, clevel, checksum_version, use_pagemap, + &headers, backup_mode); } /* check for errors */ @@ -690,6 +638,142 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, pg_free(headers); } +/* + * Catchup data file in the from_root directory to the to_root directory with + * same relative path. If sync_lsn is not NULL, only pages with equal or + * higher lsn will be copied. + * Not just copy file, but read it block by block (use bitmap in case of + * incremental catchup), validate page checksum. + */ +void +catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr sync_lsn, BackupMode backup_mode, + uint32 checksum_version, size_t prev_size) +{ + int rc; + bool use_pagemap; + char *errmsg = NULL; + BlockNumber err_blknum = 0; + + /* + * Compute expected number of blocks in the file. + * NOTE This is a normal situation, if the file size has changed + * since the moment we computed it. + */ + file->n_blocks = file->size/BLCKSZ; + + /* + * Skip unchanged file only if it exists in destination directory. + * This way we can correctly handle null-sized files which are + * not tracked by pagemap and thus always marked as unchanged. + */ + if (backup_mode == BACKUP_MODE_DIFF_PTRACK && + file->pagemap.bitmapsize == PageBitmapIsEmpty && + file->exists_in_prev && file->size == prev_size && !file->pagemap_isabsent) + { + /* + * There are none changed pages. + */ + file->write_size = BYTES_INVALID; + return; + } + + /* reset size summary */ + file->read_size = 0; + file->write_size = 0; + file->uncompressed_size = 0; + + /* + * If page map is empty or file is not present in destination directory, + * then copy backup all pages of the relation. + */ + + if (file->pagemap.bitmapsize == PageBitmapIsEmpty || + file->pagemap_isabsent || !file->exists_in_prev || + !file->pagemap.bitmap) + use_pagemap = false; + else + use_pagemap = true; + + if (use_pagemap) + elog(LOG, "Using pagemap for file \"%s\"", file->rel_path); + + /* Remote mode */ + if (fio_is_remote(FIO_DB_HOST)) + { + rc = fio_copy_pages(to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, + NONE_COMPRESS, 1, checksum_version, + /* send pagemap if any */ + use_pagemap, + /* variables for error reporting */ + &err_blknum, &errmsg); + } + else + { + /* TODO: stop handling errors internally */ + rc = copy_pages(to_fullpath, from_fullpath, file, + /* send prev backup START_LSN */ + ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && + file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, + checksum_version, use_pagemap, backup_mode); + } + + /* check for errors */ + if (rc == FILE_MISSING) + { + elog(LOG, "File not found: \"%s\"", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; + } + + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write block %u of \"%s\": %s", + err_blknum, to_fullpath, strerror(errno)); + + else if (rc == PAGE_CORRUPTION) + { + if (errmsg) + elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", + from_fullpath, err_blknum, errmsg); + else + elog(ERROR, "Corruption detected in file \"%s\", block %u", + from_fullpath, err_blknum); + } + /* OPEN_FAILED and READ_FAILED */ + else if (rc == OPEN_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot open file \"%s\"", from_fullpath); + } + else if (rc == READ_FAILED) + { + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot read file \"%s\"", from_fullpath); + } + + file->read_size = rc * BLCKSZ; + + /* Determine that file didn`t changed in case of incremental catchup */ + if (backup_mode != BACKUP_MODE_FULL && + file->exists_in_prev && + file->write_size == 0 && + file->n_blocks > 0) + { + file->write_size = BYTES_INVALID; + } + +cleanup: + pg_free(errmsg); + pg_free(file->pagemap.bitmap); +} + /* * Backup non data file * We do not apply compression to this file. @@ -698,9 +782,9 @@ backup_data_file(ConnectionArgs* conn_arg, pgFile *file, */ void backup_non_data_file(pgFile *file, pgFile *prev_file, - const char *from_fullpath, const char *to_fullpath, - BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok) + const char *from_fullpath, const char *to_fullpath, + BackupMode backup_mode, time_t parent_backup_time, + bool missing_ok) { /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) @@ -711,24 +795,34 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, } /* - * If nonedata file exists in previous backup + * If non-data file exists in previous backup * and its mtime is less than parent backup start time ... */ - if (prev_file && file->exists_in_prev && - file->mtime <= parent_backup_time) + if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) && + (prev_file && file->exists_in_prev && + file->size == prev_file->size && + file->mtime <= parent_backup_time)) { - - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + /* + * file could be deleted under our feets. + * But then backup_non_data_file_internal will handle it safely + */ + if (file->forkName != cfm) + file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); + else + file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) { file->write_size = BYTES_INVALID; + /* get full size from previous backup for unchanged file */ + file->uncompressed_size = prev_file->uncompressed_size; return; /* ...skip copying file. */ } } backup_non_data_file_internal(from_fullpath, FIO_DB_HOST, - to_fullpath, file, true); + to_fullpath, file, missing_ok); } /* @@ -772,7 +866,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, /* page headers */ BackupPageHeader2 *headers = NULL; - pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); + pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); if (use_bitmap) backup_seq++; @@ -780,7 +874,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, backup_seq--; /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); + res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); tmp_file = (res_file) ? *res_file : NULL; /* Destination file is not exists yet at this moment */ @@ -832,13 +926,13 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, * copy the file from backup. */ total_write_len += restore_data_file_internal(in, out, tmp_file, - parse_program_version(backup->program_version), - from_fullpath, to_fullpath, dest_file->n_blocks, - use_bitmap ? &(dest_file)->pagemap : NULL, - checksum_map, backup->checksum_version, - /* shiftmap can be used only if backup state precedes the shift */ - backup->stop_lsn <= shift_lsn ? lsn_map : NULL, - headers); + parse_program_version(backup->program_version), + from_fullpath, to_fullpath, dest_file->n_blocks, + use_bitmap ? &(dest_file)->pagemap : NULL, + checksum_map, backup->checksum_version, + /* shiftmap can be used only if backup state precedes the shift */ + backup->stop_lsn <= shift_lsn ? lsn_map : NULL, + headers); if (fclose(in) != 0) elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, @@ -864,15 +958,15 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, */ size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map, BackupPageHeader2 *headers) + const char *from_fullpath, const char *to_fullpath, int nblocks, + datapagemap_t *map, PageState *checksum_map, int checksum_version, + datapagemap_t *lsn_map, BackupPageHeader2 *headers) { BlockNumber blknum = 0; - int n_hdr = -1; - size_t write_len = 0; - off_t cur_pos_out = 0; - off_t cur_pos_in = 0; + int n_hdr = -1; + size_t write_len = 0; + off_t cur_pos_out = 0; + off_t cur_pos_in = 0; /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); @@ -888,7 +982,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * but should never happen in case of blocks from FULL backup. */ if (fio_fseek(out, cur_pos_out) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", + elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); for (;;) @@ -901,7 +995,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers bool is_compressed = false; /* incremental restore vars */ - uint16 page_crc = 0; + uint16 page_crc = 0; XLogRecPtr page_lsn = InvalidXLogRecPtr; /* check for interrupt */ @@ -953,7 +1047,6 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * Now we have to deal with backward compatibility. */ read_len = MAXALIGN(compressed_size); - } else break; @@ -1064,8 +1157,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * page_may_be_compressed() function. */ if (compressed_size != BLCKSZ - || page_may_be_compressed(page.data, file->compress_alg, - backup_version)) + || page_may_be_compressed(page.data, file->compress_alg, backup_version)) { is_compressed = true; } @@ -1086,13 +1178,14 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers cur_pos_out = write_pos; } - /* If page is compressed and restore is in remote mode, send compressed - * page to the remote side. + /* + * If page is compressed and restore is in remote mode, + * send compressed page to the remote side. */ if (is_compressed) { ssize_t rc; - rc = fio_fwrite_compressed(out, page.data, compressed_size, file->compress_alg); + rc = fio_fwrite_async_compressed(out, page.data, compressed_size, file->compress_alg); if (!fio_is_remote_file(out) && rc != BLCKSZ) elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", @@ -1100,7 +1193,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers } else { - if (fio_fwrite(out, page.data, BLCKSZ) != BLCKSZ) + if (fio_fwrite_async(out, page.data, BLCKSZ) != BLCKSZ) elog(ERROR, "Cannot write block %u of \"%s\": %s", blknum, to_fullpath, strerror(errno)); } @@ -1113,7 +1206,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers datapagemap_add(map, blknum); } - elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); + elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); return write_len; } @@ -1124,10 +1217,10 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers */ void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath) + const char *from_fullpath, const char *to_fullpath) { - size_t read_len = 0; - char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ + size_t read_len = 0; + char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ /* copy content */ for (;;) @@ -1136,7 +1229,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, /* check for interrupt */ if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during nonedata file restore"); + elog(ERROR, "Interrupted during non-data file restore"); read_len = fread(buf, 1, STDIO_BUFSIZE, in); @@ -1146,7 +1239,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, if (read_len > 0) { - if (fio_fwrite(out, buf, read_len) != read_len) + if (fio_fwrite_async(out, buf, read_len) != read_len) elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); } @@ -1157,7 +1250,7 @@ restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, pg_free(buf); - elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); + elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); } size_t @@ -1190,7 +1283,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, tmp_backup = dest_backup->parent_backup_link; while (tmp_backup) { - pgFile **res_file = NULL; + pgFile **res_file = NULL; /* lookup file in intermediate backup */ res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); @@ -1202,8 +1295,8 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, */ if (!tmp_file) { - elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", - dest_file->rel_path, base36enc(tmp_backup->start_time)); + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + dest_file->rel_path, backup_id_of(tmp_backup)); continue; } @@ -1227,27 +1320,32 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, /* sanity */ if (!tmp_backup) - elog(ERROR, "Failed to locate a backup containing full copy of nonedata file \"%s\"", + elog(ERROR, "Failed to locate a backup containing full copy of non-data file \"%s\"", to_fullpath); if (!tmp_file) - elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", to_fullpath); + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of nonedata file has invalid size: %li. " + elog(ERROR, "Full copy of non-data file has invalid size: %li. " "Metadata corruption in backup %s in file: \"%s\"", - tmp_file->write_size, base36enc(tmp_backup->start_time), + tmp_file->write_size, backup_id_of(tmp_backup), to_fullpath); /* incremental restore */ if (already_exists) { /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + pg_crc32 file_crc; + if (tmp_file->forkName == cfm && + tmp_file->uncompressed_size > tmp_file->write_size) + file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false); + else + file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); if (file_crc == tmp_file->crc) { - elog(VERBOSE, "Already existing nonedata file \"%s\" has the same checksum, skip restore", + elog(LOG, "Already existing non-data file \"%s\" has the same checksum, skip restore", to_fullpath); return 0; } @@ -1275,7 +1373,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, strerror(errno)); - /* disable stdio buffering for nonedata files */ + /* disable stdio buffering for non-data files */ setvbuf(in, NULL, _IONBF, BUFSIZ); /* do actual work */ @@ -1300,10 +1398,12 @@ backup_non_data_file_internal(const char *from_fullpath, const char *to_fullpath, pgFile *file, bool missing_ok) { - FILE *in = NULL; - FILE *out = NULL; - ssize_t read_len = 0; - char *buf = NULL; + FILE *out = NULL; + char *errmsg = NULL; + int rc; + bool cut_zero_tail; + + cut_zero_tail = file->forkName == cfm; INIT_FILE_CRC32(true, file->crc); @@ -1325,107 +1425,44 @@ backup_non_data_file_internal(const char *from_fullpath, /* backup remote file */ if (fio_is_remote(FIO_DB_HOST)) - { - char *errmsg = NULL; - int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg); + rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg); + else + rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg); - /* handle errors */ - if (rc == FILE_MISSING) - { - /* maybe deleted, it's not error in case of backup */ - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); - else if (rc != SEND_OK) + /* handle errors */ + if (rc == FILE_MISSING) + { + /* maybe deleted, it's not error in case of backup */ + if (missing_ok) { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); + elog(LOG, "File \"%s\" is not found", from_fullpath); + file->write_size = FILE_NOT_FOUND; + goto cleanup; } - - pg_free(errmsg); + else + elog(ERROR, "File \"%s\" is not found", from_fullpath); } - /* backup local file */ - else + else if (rc == WRITE_FAILED) + elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); + else if (rc != SEND_OK) { - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* maybe deleted, it's not error in case of backup */ - if (errno == ENOENT) - { - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, - strerror(errno)); - } - - /* disable stdio buffering for local input/output files to avoid triple buffering */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); - - /* allocate 64kB buffer */ - buf = pgut_malloc(CHUNK_SIZE); - - /* copy content and calc CRC */ - for (;;) - { - read_len = fread(buf, 1, CHUNK_SIZE, in); - - if (ferror(in)) - elog(ERROR, "Cannot read from file \"%s\": %s", - from_fullpath, strerror(errno)); - - if (read_len > 0) - { - if (fwrite(buf, 1, read_len, out) != read_len) - elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath, - strerror(errno)); - - /* update CRC */ - COMP_FILE_CRC32(true, file->crc, buf, read_len); - file->read_size += read_len; - } - - if (feof(in)) - break; - } + if (errmsg) + elog(ERROR, "%s", errmsg); + else + elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); } - file->write_size = (int64) file->read_size; - - if (file->write_size > 0) - file->uncompressed_size = file->write_size; + file->uncompressed_size = file->read_size; cleanup: + if (errmsg != NULL) + pg_free(errmsg); + /* finish CRC calculation and store into pgFile */ FIN_FILE_CRC32(true, file->crc); - if (in && fclose(in)) - elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); - if (out && fclose(out)) elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); - - pg_free(buf); } /* @@ -1433,7 +1470,7 @@ backup_non_data_file_internal(const char *from_fullpath, */ bool create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file) + fio_location to_location, pgFile *file) { char to_path[MAXPGPATH]; FILE *out; @@ -1516,7 +1553,7 @@ validate_one_page(Page page, BlockNumber absolute_blkno, } /* - * Valiate pages of datafile in PGDATA one by one. + * Validate pages of datafile in PGDATA one by one. * * returns true if the file is valid * also returns true if the file was not found @@ -1530,7 +1567,7 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, BlockNumber nblocks = 0; int page_state; char curr_page[BLCKSZ]; - bool is_valid = true; + bool is_valid = true; in = fopen(from_fullpath, PG_BINARY_R); if (in == NULL) @@ -1563,10 +1600,10 @@ check_data_file(ConnectionArgs *arguments, pgFile *file, for (blknum = 0; blknum < nblocks; blknum++) { PageState page_st; - page_state = prepare_page(NULL, file, InvalidXLogRecPtr, - blknum, in, BACKUP_MODE_FULL, - curr_page, false, checksum_version, - 0, NULL, from_fullpath, &page_st); + page_state = prepare_page(file, InvalidXLogRecPtr, + blknum, in, BACKUP_MODE_FULL, + curr_page, false, checksum_version, + from_fullpath, &page_st); if (page_state == PageIsTruncated) break; @@ -1599,7 +1636,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, int n_hdr = -1; off_t cur_pos_in = 0; - elog(VERBOSE, "Validate relation blocks for file \"%s\"", fullpath); + elog(LOG, "Validate relation blocks for file \"%s\"", fullpath); /* should not be possible */ Assert(!(backup_version >= 20400 && file->n_headers <= 0)); @@ -1624,9 +1661,9 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, while (true) { int rc = 0; - size_t len = 0; + size_t len = 0; DataPage compressed_page; /* used as read buffer */ - int compressed_size = 0; + int compressed_size = 0; DataPage page; BlockNumber blknum = 0; PageState page_st; @@ -1658,7 +1695,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, elog(ERROR, "Cannot seek block %u of \"%s\": %s", blknum, fullpath, strerror(errno)); else - elog(INFO, "Seek to %u", headers[n_hdr].pos); + elog(VERBOSE, "Seek to %u", headers[n_hdr].pos); cur_pos_in = headers[n_hdr].pos; } @@ -1682,7 +1719,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, /* backward compatibility kludge TODO: remove in 3.0 */ if (compressed_size == PageIsTruncated) { - elog(INFO, "Block %u of \"%s\" is truncated", + elog(VERBOSE, "Block %u of \"%s\" is truncated", blknum, fullpath); continue; } @@ -1714,7 +1751,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, || page_may_be_compressed(compressed_page.data, file->compress_alg, backup_version)) { - int32 uncompressed_size = 0; + int32 uncompressed_size = 0; const char *errormsg = NULL; uncompressed_size = do_decompress(page.data, BLCKSZ, @@ -1742,21 +1779,21 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } rc = validate_one_page(page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); } else rc = validate_one_page(compressed_page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); + file->segno * RELSEG_SIZE + blknum, + stop_lsn, &page_st, checksum_version); switch (rc) { case PAGE_IS_NOT_FOUND: - elog(LOG, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); + elog(VERBOSE, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); break; case PAGE_IS_ZEROED: - elog(LOG, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); + elog(VERBOSE, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); break; case PAGE_HEADER_IS_INVALID: elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); @@ -1851,7 +1888,7 @@ get_checksum_map(const char *fullpath, uint32 checksum_version, if (feof(in)) break; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during page reading"); } @@ -1866,11 +1903,11 @@ datapagemap_t * get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) { - FILE *in = NULL; - BlockNumber blknum = 0; - char read_buffer[BLCKSZ]; - char in_buf[STDIO_BUFSIZE]; - datapagemap_t *lsn_map = NULL; + FILE *in = NULL; + BlockNumber blknum = 0; + char read_buffer[BLCKSZ]; + char in_buf[STDIO_BUFSIZE]; + datapagemap_t *lsn_map = NULL; Assert(shift_lsn > 0); @@ -1914,7 +1951,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, if (feof(in)) break; - if (interrupted) + if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during page reading"); } @@ -1948,11 +1985,11 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, return false; /* EOF found */ else if (read_len != 0 && feof(in)) elog(ERROR, - "Odd size page found at offset %lu of \"%s\"", - ftell(in), fullpath); + "Odd size page found at offset %ld of \"%s\"", + ftello(in), fullpath); else - elog(ERROR, "Cannot read header at offset %lu of \"%s\": %s", - ftell(in), fullpath, strerror(errno)); + elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", + ftello(in), fullpath, strerror(errno)); } /* In older versions < 2.4.0, when crc for file was calculated, header was @@ -1994,20 +2031,21 @@ open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) /* backup local file */ int -send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, +send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema) + BackupMode backup_mode) { FILE *in = NULL; FILE *out = NULL; - int hdr_num = -1; off_t cur_pos_out = 0; char curr_page[BLCKSZ]; int n_blocks_read = 0; BlockNumber blknum = 0; datapagemap_iterator_t *iter = NULL; int compressed_size = 0; + BackupPageHeader2 *header = NULL; + parray *harray = NULL; /* stdio buffers */ char *in_buf = NULL; @@ -2046,14 +2084,16 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); } + harray = parray_new(); + while (blknum < file->n_blocks) { PageState page_st; - int rc = prepare_page(conn_arg, file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - ptrack_version_num, ptrack_schema, - from_fullpath, &page_st); + int rc = prepare_page(file, prev_backup_start_lsn, + blknum, in, backup_mode, curr_page, + true, checksum_version, + from_fullpath, &page_st); + if (rc == PageIsTruncated) break; @@ -2063,17 +2103,15 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f if (!out) out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - hdr_num++; - - if (!*headers) - *headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+1) * sizeof(BackupPageHeader2)); + header = pgut_new0(BackupPageHeader2); + *header = (BackupPageHeader2){ + .block = blknum, + .pos = cur_pos_out, + .lsn = page_st.lsn, + .checksum = page_st.checksum, + }; - (*headers)[hdr_num].block = blknum; - (*headers)[hdr_num].pos = cur_pos_out; - (*headers)[hdr_num].lsn = page_st.lsn; - (*headers)[hdr_num].checksum = page_st.checksum; + parray_append(harray, header); compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), rc, curr_page, calg, clevel, @@ -2098,12 +2136,22 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f * Add dummy header, so we can later extract the length of last header * as difference between their offsets. */ - if (*headers) + if (parray_num(harray) > 0) { - file->n_headers = hdr_num +1; - *headers = (BackupPageHeader2 *) pgut_realloc(*headers, (hdr_num+2) * sizeof(BackupPageHeader2)); - (*headers)[hdr_num+1].pos = cur_pos_out; + size_t hdr_num = parray_num(harray); + size_t i; + + file->n_headers = (int) hdr_num; /* is it valid? */ + *headers = (BackupPageHeader2 *) pgut_malloc0((hdr_num + 1) * sizeof(BackupPageHeader2)); + for (i = 0; i < hdr_num; i++) + { + header = (BackupPageHeader2 *)parray_get(harray, i); + (*headers)[i] = *header; + pg_free(header); + } + (*headers)[hdr_num] = (BackupPageHeader2){.pos=cur_pos_out}; } + parray_free(harray); /* cleanup */ if (in && fclose(in)) @@ -2122,6 +2170,147 @@ send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_f return n_blocks_read; } +/* + * Copy local data file just as send_pages but without attaching additional header and compression + */ +int +copy_pages(const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr sync_lsn, + uint32 checksum_version, bool use_pagemap, + BackupMode backup_mode) +{ + FILE *in = NULL; + FILE *out = NULL; + char curr_page[BLCKSZ]; + int n_blocks_read = 0; + BlockNumber blknum = 0; + datapagemap_iterator_t *iter = NULL; + + /* stdio buffers */ + char *in_buf = NULL; + char *out_buf = NULL; + + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* + * If file is not found, this is not en error. + * It could have been deleted by concurrent postgres transaction. + */ + if (errno == ENOENT) + return FILE_MISSING; + + elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); + } + + /* + * Enable stdio buffering for local input file, + * unless the pagemap is involved, which + * imply a lot of random access. + */ + + if (use_pagemap) + { + iter = datapagemap_iterate(&file->pagemap); + datapagemap_next(iter, &blknum); /* set first block */ + + setvbuf(in, NULL, _IONBF, BUFSIZ); + } + else + { + in_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); + } + + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + if (out == NULL) + elog(ERROR, "Cannot open destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* update file permission */ + if (chmod(to_fullpath, file->mode) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + /* Enable buffering for output file */ + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + + while (blknum < file->n_blocks) + { + PageState page_st; + int rc = prepare_page(file, sync_lsn, + blknum, in, backup_mode, curr_page, + true, checksum_version, + from_fullpath, &page_st); + if (rc == PageIsTruncated) + break; + + else if (rc == PageIsOk) + { + if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) + elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", + blknum * BLCKSZ, to_fullpath, strerror(errno)); + + if (write_page(file, out, curr_page) != BLCKSZ) + elog(ERROR, "File: \"%s\", cannot write at block %u: %s", + to_fullpath, blknum, strerror(errno)); + } + + n_blocks_read++; + + /* next block */ + if (use_pagemap) + { + /* exit if pagemap is exhausted */ + if (!datapagemap_next(iter, &blknum)) + break; + } + else + blknum++; + } + + /* truncate output file if required */ + if (fseek(out, 0, SEEK_END) != 0) + elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", + to_fullpath, strerror(errno)); + { + long pos = ftell(out); + + if (pos < 0) + elog(ERROR, "Cannot get position in destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + if (pos != file->size) + { + if (fflush(out) != 0) + elog(ERROR, "Cannot flush destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + if (ftruncate(fileno(out), file->size) == -1) + elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + to_fullpath, file->size, strerror(errno)); + } + } + + /* cleanup */ + if (fclose(in)) + elog(ERROR, "Cannot close the source file \"%s\": %s", + to_fullpath, strerror(errno)); + + /* close output file */ + if (fclose(out)) + elog(ERROR, "Cannot close the destination file \"%s\": %s", + to_fullpath, strerror(errno)); + + pg_free(iter); + pg_free(in_buf); + pg_free(out_buf); + + return n_blocks_read; +} + /* * Attempt to open header file, read content and return as * array of headers. @@ -2156,11 +2345,11 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b return NULL; } /* disable buffering for header file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); + setvbuf(in, NULL, _IONBF, 0); - if (fseek(in, file->hdr_off, SEEK_SET)) + if (fseeko(in, file->hdr_off, SEEK_SET)) { - elog(strict ? ERROR : WARNING, "Cannot seek to position %lu in page header map \"%s\": %s", + elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", file->hdr_off, hdr_map->path, strerror(errno)); goto cleanup; } @@ -2177,7 +2366,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) { - elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %li len: %i \"%s\": %s", + elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); goto cleanup; } @@ -2208,7 +2397,7 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b if (hdr_crc != file->hdr_crc) { elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " - "offset: %lu, len: %lu, current: %u, expected: %u", + "offset: %llu, len: %lu, current: %u, expected: %u", file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); goto cleanup; } @@ -2230,6 +2419,8 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b return headers; } +/* write headers of all blocks belonging to file to header map and + * save its offset and size */ void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge) { @@ -2245,19 +2436,19 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, /* when running merge we must write headers into temp map */ map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; - read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); + read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); /* calculate checksums */ INIT_FILE_CRC32(true, file->hdr_crc); COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); FIN_FILE_CRC32(true, file->hdr_crc); - zheaders = pgut_malloc(read_len*2); - memset(zheaders, 0, read_len*2); + zheaders = pgut_malloc(read_len * 2); + memset(zheaders, 0, read_len * 2); /* compress headers */ - z_len = do_compress(zheaders, read_len*2, headers, - read_len, ZLIB_COMPRESS, 1, &errormsg); + z_len = do_compress(zheaders, read_len * 2, headers, + read_len, ZLIB_COMPRESS, 1, &errormsg); /* writing to header map must be serialized */ pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ @@ -2266,7 +2457,7 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, { elog(LOG, "Creating page header map \"%s\"", map_path); - hdr_map->fp = fopen(map_path, PG_BINARY_W); + hdr_map->fp = fopen(map_path, PG_BINARY_A); if (hdr_map->fp == NULL) elog(ERROR, "Cannot open header file \"%s\": %s", map_path, strerror(errno)); @@ -2295,13 +2486,16 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, file->rel_path, z_len); } - elog(VERBOSE, "Writing headers for file \"%s\" offset: %li, len: %i, crc: %u", + elog(VERBOSE, "Writing headers for file \"%s\" offset: %llu, len: %i, crc: %u", file->rel_path, file->hdr_off, z_len, file->hdr_crc); if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) + { + pthread_mutex_unlock(&(hdr_map->mutex)); elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); + } - file->hdr_size = z_len; /* save the length of compressed headers */ + file->hdr_size = z_len; /* save the length of compressed headers */ hdr_map->offset += z_len; /* update current offset in map */ /* End critical section */ diff --git a/src/datapagemap.c b/src/datapagemap.c new file mode 100644 index 000000000..7e4202a72 --- /dev/null +++ b/src/datapagemap.c @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * datapagemap.c + * A data structure for keeping track of data pages that have changed. + * + * This is a fairly simple bitmap. + * + * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include "datapagemap.h" + +struct datapagemap_iterator +{ + datapagemap_t *map; + BlockNumber nextblkno; +}; + +/***** + * Public functions + */ + +/* + * Add a block to the bitmap. + */ +void +datapagemap_add(datapagemap_t *map, BlockNumber blkno) +{ + int offset; + int bitno; + int oldsize = map->bitmapsize; + + offset = blkno / 8; + bitno = blkno % 8; + + /* enlarge or create bitmap if needed */ + if (oldsize <= offset) + { + int newsize; + + /* + * The minimum to hold the new bit is offset + 1. But add some + * headroom, so that we don't need to repeatedly enlarge the bitmap in + * the common case that blocks are modified in order, from beginning + * of a relation to the end. + */ + newsize = (oldsize == 0) ? 16 : oldsize; + while (newsize <= offset) { + newsize <<= 1; + } + + map->bitmap = pg_realloc(map->bitmap, newsize); + + /* zero out the newly allocated region */ + memset(&map->bitmap[oldsize], 0, newsize - oldsize); + + map->bitmapsize = newsize; + } + + /* Set the bit */ + map->bitmap[offset] |= (1 << bitno); +} + +/* + * Start iterating through all entries in the page map. + * + * After datapagemap_iterate, call datapagemap_next to return the entries, + * until it returns false. After you're done, use pg_free() to destroy the + * iterator. + */ +datapagemap_iterator_t * +datapagemap_iterate(datapagemap_t *map) +{ + datapagemap_iterator_t *iter; + + iter = pg_malloc(sizeof(datapagemap_iterator_t)); + iter->map = map; + iter->nextblkno = 0; + + return iter; +} + +bool +datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno) +{ + datapagemap_t *map = iter->map; + + for (;;) + { + BlockNumber blk = iter->nextblkno; + int nextoff = blk / 8; + int bitno = blk % 8; + + if (nextoff >= map->bitmapsize) + break; + + iter->nextblkno++; + + if (map->bitmap[nextoff] & (1 << bitno)) + { + *blkno = blk; + return true; + } + } + + /* no more set bits in this bitmap. */ + return false; +} + diff --git a/src/datapagemap.h b/src/datapagemap.h new file mode 100644 index 000000000..6ad7a6204 --- /dev/null +++ b/src/datapagemap.h @@ -0,0 +1,34 @@ +/*------------------------------------------------------------------------- + * + * datapagemap.h + * + * Copyright (c) 2013-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ +#ifndef DATAPAGEMAP_H +#define DATAPAGEMAP_H + +#if PG_VERSION_NUM < 160000 +#include "storage/relfilenode.h" +#else +#include "storage/relfilelocator.h" +#define RelFileNode RelFileLocator +#endif +#include "storage/block.h" + + +struct datapagemap +{ + char *bitmap; + int bitmapsize; +}; + +typedef struct datapagemap datapagemap_t; +typedef struct datapagemap_iterator datapagemap_iterator_t; + +extern void datapagemap_add(datapagemap_t *map, BlockNumber blkno); +extern datapagemap_iterator_t *datapagemap_iterate(datapagemap_t *map); +extern bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno); + +#endif /* DATAPAGEMAP_H */ diff --git a/src/delete.c b/src/delete.c index b3c50a4b9..f48ecc95f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -14,14 +14,15 @@ #include #include -static void delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tli, +static void delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tli, uint32 xlog_seg_size, bool dry_run); static void do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list); -static void do_retention_merge(parray *backup_list, parray *to_keep_list, - parray *to_purge_list); +static void do_retention_merge(InstanceState *instanceState, parray *backup_list, + parray *to_keep_list, parray *to_purge_list, + bool no_validate, bool no_sync); static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); -static void do_retention_wal(bool dry_run); +static void do_retention_wal(InstanceState *instanceState, bool dry_run); // TODO: more useful messages for dry run. static bool backup_deleted = false; /* At least one backup was deleted */ @@ -29,17 +30,17 @@ static bool backup_merged = false; /* At least one merge was enacted */ static bool wal_deleted = false; /* At least one WAL segments was deleted */ void -do_delete(time_t backup_id) +do_delete(InstanceState *instanceState, time_t backup_id) { int i; parray *backup_list, *delete_list; pgBackup *target_backup = NULL; - size_t size_to_delete = 0; + int64 size_to_delete = 0; char size_to_delete_pretty[20]; /* Get complete list of backups */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); delete_list = parray_new(); @@ -70,7 +71,7 @@ do_delete(time_t backup_id) parray_append(delete_list, backup); elog(LOG, "Backup %s %s be deleted", - base36enc(backup->start_time), dry_run? "can":"will"); + backup_id_of(backup), dry_run? "can":"will"); size_to_delete += backup->data_bytes; if (backup->stream) @@ -83,13 +84,13 @@ do_delete(time_t backup_id) { pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); elog(INFO, "Resident data size to free by delete of backup %s : %s", - base36enc(target_backup->start_time), size_to_delete_pretty); + backup_id_of(target_backup), size_to_delete_pretty); } if (!dry_run) { /* Lock marked for delete backups */ - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false); + catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false, true); /* Delete backups from the end of list */ for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) @@ -105,7 +106,7 @@ do_delete(time_t backup_id) /* Clean WAL segments */ if (delete_wal) - do_retention_wal(dry_run); + do_retention_wal(instanceState, dry_run); /* cleanup */ parray_free(delete_list); @@ -123,7 +124,7 @@ do_delete(time_t backup_id) * which FULL backup should be keeped for redundancy obligation(only valid do), * but if invalid backup is not guarded by retention - it is removed */ -void do_retention(void) +void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync) { parray *backup_list = NULL; parray *to_keep_list = parray_new(); @@ -135,8 +136,11 @@ void do_retention(void) backup_deleted = false; backup_merged = false; + /* For now retention is possible only locally */ + MyLocation = FIO_LOCAL_HOST; + /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) backup_list_is_empty = true; @@ -154,7 +158,13 @@ void do_retention(void) /* Retention is disabled but we still can cleanup wal */ elog(WARNING, "Retention policy is not set"); if (!delete_wal) + { + parray_walk(backup_list, pgBackupFree); + parray_free(backup_list); + parray_free(to_keep_list); + parray_free(to_purge_list); return; + } } else /* At least one retention policy is active */ @@ -169,14 +179,14 @@ void do_retention(void) do_retention_internal(backup_list, to_keep_list, to_purge_list); if (merge_expired && !dry_run && !backup_list_is_empty) - do_retention_merge(backup_list, to_keep_list, to_purge_list); + do_retention_merge(instanceState, backup_list, to_keep_list, to_purge_list, no_validate, no_sync); if (delete_expired && !dry_run && !backup_list_is_empty) do_retention_purge(to_keep_list, to_purge_list); /* TODO: some sort of dry run for delete_wal */ if (delete_wal) - do_retention_wal(dry_run); + do_retention_wal(instanceState, dry_run); /* TODO: consider dry-run flag */ @@ -224,22 +234,23 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg { pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - /* Consider only valid FULL backups for Redundancy */ - if (instance_config.retention_redundancy > 0 && - backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) + if (backup->backup_mode == BACKUP_MODE_FULL) { - n_full_backups++; - /* Add every FULL backup that satisfy Redundancy policy to separate list */ - if (n_full_backups <= instance_config.retention_redundancy) + if (n_full_backups < instance_config.retention_redundancy) { if (!redundancy_full_backup_list) redundancy_full_backup_list = parray_new(); parray_append(redundancy_full_backup_list, backup); } + + /* Consider only valid FULL backups for Redundancy fulfillment */ + if (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE) + { + n_full_backups++; + } } } /* Sort list of full backups to keep */ @@ -313,15 +324,15 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg (backup->expire_time > current_time)) { char expire_timestamp[100]; - time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time); + time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time, false); elog(LOG, "Backup %s is pinned until '%s', retain", - base36enc(backup->start_time), expire_timestamp); + backup_id_of(backup), expire_timestamp); continue; } /* Add backup to purge_list */ - elog(VERBOSE, "Mark backup %s for purge.", base36enc(backup->start_time)); + elog(VERBOSE, "Mark backup %s for purge.", backup_id_of(backup)); parray_append(to_purge_list, backup); continue; } @@ -401,8 +412,8 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg /* TODO: add ancestor(chain full backup) ID */ elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", - base36enc(backup->start_time), - pgBackupGetBackupMode(backup), + backup_id_of(backup), + pgBackupGetBackupMode(backup, false), status2str(backup->status), cur_full_backup_num, instance_config.retention_redundancy, @@ -410,14 +421,19 @@ do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purg pinning_window ? pinning_window : instance_config.retention_window, action); - if (backup->backup_mode == BACKUP_MODE_FULL) + /* Only valid full backups are count to something */ + if (backup->backup_mode == BACKUP_MODE_FULL && + (backup->status == BACKUP_STATUS_OK || + backup->status == BACKUP_STATUS_DONE)) cur_full_backup_num++; } } /* Merge partially expired incremental chains */ static void -do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_list) +do_retention_merge(InstanceState *instanceState, parray *backup_list, + parray *to_keep_list, parray *to_purge_list, + bool no_validate, bool no_sync) { int i; int j; @@ -441,7 +457,6 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l /* Merging happens here */ for (i = 0; i < parray_num(to_keep_list); i++) { - char *keep_backup_id = NULL; pgBackup *full_backup = NULL; parray *merge_list = NULL; @@ -451,7 +466,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l if (!keep_backup) continue; - elog(INFO, "Consider backup %s for merge", base36enc(keep_backup->start_time)); + elog(INFO, "Consider backup %s for merge", backup_id_of(keep_backup)); /* Got valid incremental backup, find its FULL ancestor */ full_backup = find_parent_full_backup(keep_backup); @@ -459,7 +474,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l /* Failed to find parent */ if (!full_backup) { - elog(WARNING, "Failed to find FULL parent for %s", base36enc(keep_backup->start_time)); + elog(WARNING, "Failed to find FULL parent for %s", backup_id_of(keep_backup)); continue; } @@ -469,7 +484,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l pgBackupCompareIdDesc)) { elog(WARNING, "Skip backup %s for merging, " - "because his FULL parent is not marked for purge", base36enc(keep_backup->start_time)); + "because his FULL parent is not marked for purge", backup_id_of(keep_backup)); continue; } @@ -478,10 +493,9 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l * backups from purge_list. */ - keep_backup_id = base36enc_dup(keep_backup->start_time); elog(INFO, "Merge incremental chain between full backup %s and backup %s", - base36enc(full_backup->start_time), keep_backup_id); - pg_free(keep_backup_id); + backup_id_of(full_backup), + backup_id_of(keep_backup)); merge_list = parray_new(); @@ -510,7 +524,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l parray_rm(to_purge_list, full_backup, pgBackupCompareId); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* Consider this extreme case */ // PAGEa1 PAGEb1 both valid @@ -523,7 +537,7 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l // if (is_prolific(backup_list, full_backup)) // { // elog(WARNING, "Backup %s has multiple valid descendants. " -// "Automatic merge is not possible.", base36enc(full_backup->start_time)); +// "Automatic merge is not possible.", backup_id_of(full_backup)); // } /* Merge list example: @@ -534,9 +548,8 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l * * Merge incremental chain from PAGE3 into FULL. */ - keep_backup = parray_get(merge_list, 0); - merge_chain(merge_list, full_backup, keep_backup); + merge_chain(instanceState, merge_list, full_backup, keep_backup, no_validate, no_sync); backup_merged = true; for (j = parray_num(merge_list) - 2; j >= 0; j--) @@ -545,12 +558,17 @@ do_retention_merge(parray *backup_list, parray *to_keep_list, parray *to_purge_l /* Try to remove merged incremental backup from both keep and purge lists */ parray_rm(to_purge_list, tmp_backup, pgBackupCompareId); - parray_set(to_keep_list, i, NULL); + for (i = 0; i < parray_num(to_keep_list); i++) + if (parray_get(to_keep_list, i) == tmp_backup) + { + parray_set(to_keep_list, i, NULL); + break; + } } - - pgBackupValidate(full_backup, NULL); + if (!no_validate) + pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Merging of backup %s failed", base36enc(full_backup->start_time)); + elog(ERROR, "Merging of backup %s failed", backup_id_of(full_backup)); /* Cleanup */ parray_free(merge_list); @@ -582,7 +600,7 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j); elog(LOG, "Consider backup %s for purge", - base36enc(delete_backup->start_time)); + backup_id_of(delete_backup)); /* Evaluate marked for delete backup against every backup in keep list. * If marked for delete backup is recognized as parent of one of those, @@ -590,8 +608,6 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) */ for (i = 0; i < parray_num(to_keep_list); i++) { - char *keeped_backup_id; - pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); /* item could have been nullified in merge */ @@ -602,10 +618,9 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) if (keep_backup->backup_mode == BACKUP_MODE_FULL) continue; - keeped_backup_id = base36enc_dup(keep_backup->start_time); - elog(LOG, "Check if backup %s is parent of backup %s", - base36enc(delete_backup->start_time), keeped_backup_id); + backup_id_of(delete_backup), + backup_id_of(keep_backup)); if (is_parent(delete_backup->start_time, keep_backup, true)) { @@ -613,13 +628,12 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) /* We must not delete this backup, evict it from purge list */ elog(LOG, "Retain backup %s because his " "descendant %s is guarded by retention", - base36enc(delete_backup->start_time), keeped_backup_id); + backup_id_of(delete_backup), + backup_id_of(keep_backup)); purge = false; - pg_free(keeped_backup_id); break; } - pg_free(keeped_backup_id); } /* Retain backup */ @@ -627,11 +641,11 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) continue; /* Actual purge */ - if (!lock_backup(delete_backup, false)) + if (!lock_backup(delete_backup, false, true)) { /* If the backup still is used, do not interrupt and go to the next */ elog(WARNING, "Cannot lock backup %s directory, skip purging", - base36enc(delete_backup->start_time)); + backup_id_of(delete_backup)); continue; } @@ -649,12 +663,13 @@ do_retention_purge(parray *to_keep_list, parray *to_purge_list) * and delete them. */ static void -do_retention_wal(bool dry_run) +do_retention_wal(InstanceState *instanceState, bool dry_run) { parray *tli_list; int i; - tli_list = catalog_get_timelines(&instance_config); + //TODO check that instanceState is not NULL + tli_list = catalog_get_timelines(instanceState, &instance_config); for (i = 0; i < parray_num(tli_list); i++) { @@ -672,12 +687,11 @@ do_retention_wal(bool dry_run) * at least one backup and no file should be removed. * Unless wal-depth is enabled. */ - if ((tlinfo->closest_backup) && instance_config.wal_depth <= 0) + if ((tlinfo->closest_backup) && instance_config.wal_depth == 0) continue; /* WAL retention keeps this timeline from purge */ - if (instance_config.wal_depth >= 0 && tlinfo->anchor_tli > 0 && - tlinfo->anchor_tli != tlinfo->tli) + if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) continue; /* @@ -691,24 +705,24 @@ do_retention_wal(bool dry_run) */ if (tlinfo->oldest_backup) { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) { - delete_walfiles_in_tli(tlinfo->anchor_lsn, + delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); } else { - delete_walfiles_in_tli(tlinfo->oldest_backup->start_lsn, + delete_walfiles_in_tli(instanceState, tlinfo->oldest_backup->start_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); } } else { - if (instance_config.wal_depth >= 0 && !(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) - delete_walfiles_in_tli(tlinfo->anchor_lsn, + if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) + delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, tlinfo, instance_config.xlog_seg_size, dry_run); else - delete_walfiles_in_tli(InvalidXLogRecPtr, + delete_walfiles_in_tli(instanceState, InvalidXLogRecPtr, tlinfo, instance_config.xlog_seg_size, dry_run); } } @@ -717,6 +731,7 @@ do_retention_wal(bool dry_run) /* * Delete backup files of the backup and update the status of the backup to * BACKUP_STATUS_DELETED. + * TODO: delete files on multiple threads */ void delete_backup_files(pgBackup *backup) @@ -733,20 +748,23 @@ delete_backup_files(pgBackup *backup) if (backup->status == BACKUP_STATUS_DELETED) { elog(WARNING, "Backup %s already deleted", - base36enc(backup->start_time)); + backup_id_of(backup)); return; } - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + if (backup->recovery_time) + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); + else + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); elog(INFO, "Delete: %s %s", - base36enc(backup->start_time), timestamp); + backup_id_of(backup), timestamp); /* * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which * the error occurs before deleting all backup files. */ - write_backup_status(backup, BACKUP_STATUS_DELETING, instance_name, false); + write_backup_status(backup, BACKUP_STATUS_DELETING, false); /* list files to be deleted */ files = parray_new(); @@ -800,7 +818,7 @@ delete_backup_files(pgBackup *backup) * Q: Maybe we should stop treating partial WAL segments as second-class citizens? */ static void -delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, +delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tlinfo, uint32 xlog_seg_size, bool dry_run) { XLogSegNo FirstToDeleteSegNo; @@ -925,10 +943,10 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, { char wal_fullpath[MAXPGPATH]; - join_path_components(wal_fullpath, instance_config.arclog_path, wal_file->file.name); + join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name); /* save segment from purging */ - if (instance_config.wal_depth >= 0 && wal_file->keep) + if (wal_file->keep) { elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath); continue; @@ -962,17 +980,15 @@ delete_walfiles_in_tli(XLogRecPtr keep_lsn, timelineInfo *tlinfo, /* Delete all backup files and wal files of given instance. */ int -do_delete_instance(void) +do_delete_instance(InstanceState *instanceState) { parray *backup_list; int i; - char instance_config_path[MAXPGPATH]; - /* Delete all backups. */ - backup_list = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true); + catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true); for (i = 0; i < parray_num(backup_list); i++) { @@ -985,38 +1001,37 @@ do_delete_instance(void) parray_free(backup_list); /* Delete all wal files. */ - pgut_rmtree(arclog_path, false, true); + pgut_rmtree(instanceState->instance_wal_subdir_path, false, true); /* Delete backup instance config file */ - join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE); - if (remove(instance_config_path)) + if (remove(instanceState->instance_config_path)) { - elog(ERROR, "Can't remove \"%s\": %s", instance_config_path, + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_config_path, strerror(errno)); } /* Delete instance root directories */ - if (rmdir(backup_instance_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", backup_instance_path, + if (rmdir(instanceState->instance_backup_subdir_path) != 0) + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); - if (rmdir(arclog_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", arclog_path, + if (rmdir(instanceState->instance_wal_subdir_path) != 0) + elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_wal_subdir_path, strerror(errno)); - elog(INFO, "Instance '%s' successfully deleted", instance_name); + elog(INFO, "Instance '%s' successfully deleted", instanceState->instance_name); return 0; } /* Delete all backups of given status in instance */ void -do_delete_status(InstanceConfig *instance_config, const char *status) +do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, const char *status) { int i; parray *backup_list, *delete_list; const char *pretty_status; int n_deleted = 0, n_found = 0; - size_t size_to_delete = 0; + int64 size_to_delete = 0; char size_to_delete_pretty[20]; pgBackup *backup; @@ -1033,11 +1048,13 @@ do_delete_status(InstanceConfig *instance_config, const char *status) */ pretty_status = status2str(status_for_delete); - backup_list = catalog_get_backup_list(instance_config->name, INVALID_BACKUP_ID); + backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); if (parray_num(backup_list) == 0) { - elog(WARNING, "Instance '%s' has no backups", instance_config->name); + elog(WARNING, "Instance '%s' has no backups", instanceState->instance_name); + parray_free(delete_list); + parray_free(backup_list); return; } @@ -1072,13 +1089,13 @@ do_delete_status(InstanceConfig *instance_config, const char *status) backup = (pgBackup *)parray_get(delete_list, i); elog(INFO, "Backup %s with status %s %s be deleted", - base36enc(backup->start_time), status2str(backup->status), dry_run ? "can" : "will"); + backup_id_of(backup), status2str(backup->status), dry_run ? "can" : "will"); size_to_delete += backup->data_bytes; if (backup->stream) size_to_delete += backup->wal_bytes; - if (!dry_run && lock_backup(backup, false)) + if (!dry_run && lock_backup(backup, false, true)) delete_backup_files(backup); n_deleted++; @@ -1096,12 +1113,12 @@ do_delete_status(InstanceConfig *instance_config, const char *status) if (!dry_run && n_deleted > 0) elog(INFO, "Successfully deleted %i %s from instance '%s'", n_deleted, n_deleted == 1 ? "backup" : "backups", - instance_config->name); + instanceState->instance_name); if (n_found == 0) elog(WARNING, "Instance '%s' has no backups with status '%s'", - instance_config->name, pretty_status); + instanceState->instance_name, pretty_status); // we don`t do WAL purge here, because it is impossible to correctly handle // dry-run case. diff --git a/src/dir.c b/src/dir.c index dfbb6e8c4..4b1bc2816 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,11 +3,12 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ +#include #include "pg_probackup.h" #include "utils/file.h" @@ -28,7 +29,7 @@ * start so they are not included in backups. The directories themselves are * kept and included as empty to preserve access permissions. */ -const char *pgdata_exclude_dir[] = +static const char *pgdata_exclude_dir[] = { PG_XLOG_DIR, /* @@ -121,8 +122,6 @@ typedef struct TablespaceCreatedList TablespaceCreatedListCell *tail; } TablespaceCreatedList; -static int pgCompareString(const void *str1, const void *str2); - static char dir_check_file(pgFile *file, bool backup_logs); static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, @@ -130,6 +129,10 @@ static void dir_list_file_internal(parray *files, pgFile *parent, const char *pa bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); +static void cleanup_tablespace(const char *path); + +static void control_string_bad_format(const char* str); + /* Tablespace mapping */ static TablespaceList tablespace_dirs = {NULL, NULL}; @@ -138,25 +141,29 @@ static TablespaceList external_remap_list = {NULL, NULL}; /* * Create directory, also create parent directories if necessary. + * In strict mode treat already existing directory as error. + * Return values: + * 0 - ok + * -1 - error (check errno) */ int -dir_create_dir(const char *dir, mode_t mode) +dir_create_dir(const char *dir, mode_t mode, bool strict) { char parent[MAXPGPATH]; - strncpy(parent, dir, MAXPGPATH); + strlcpy(parent, dir, MAXPGPATH); get_parent_directory(parent); /* Create parent first */ - if (access(parent, F_OK) == -1) - dir_create_dir(parent, mode); + if (strlen(parent) > 0 && access(parent, F_OK) == -1) + dir_create_dir(parent, mode, false); /* Create directory */ if (mkdir(dir, mode) == -1) { - if (errno == EEXIST) /* already exist */ + if (errno == EEXIST && !strict) /* already exist */ return 0; - elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno)); + return -1; } return 0; @@ -175,7 +182,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, /* file not found is not an error case */ if (errno == ENOENT) return NULL; - elog(ERROR, "cannot stat file \"%s\": %s", path, + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); } @@ -217,6 +224,9 @@ pgFileInit(const char *rel_path) /* Number of blocks backed up during backup */ file->n_headers = 0; + // May be add? + // pg_atomic_clear_flag(file->lock); + file->excluded = false; return file; } @@ -252,137 +262,6 @@ pgFileDelete(mode_t mode, const char *full_path) } } -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) -{ - FILE *fp; - pg_crc32 crc = 0; - char *buf; - size_t len = 0; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fread(buf, 1, STDIO_BUFSIZE, fp); - - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - - if (feof(fp)) - break; - } - - FIN_FILE_CRC32(use_crc32c, crc); - fclose(fp); - pg_free(buf); - - return crc; -} - -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) -{ - gzFile fp; - pg_crc32 crc = 0; - int len = 0; - int err; - char *buf; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = gzopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = gzread(fp, buf, STDIO_BUFSIZE); - - if (len <= 0) - { - /* we either run into eof or error */ - if (gzeof(fp)) - break; - else - { - const char *err_str = NULL; - - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); - } - } - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - } - - FIN_FILE_CRC32(use_crc32c, crc); - gzclose(fp); - pg_free(buf); - - return crc; -} - void pgFileFree(void *file) { @@ -419,6 +298,26 @@ pgFileCompareName(const void *f1, const void *f2) return strcmp(f1p->name, f2p->name); } +/* Compare pgFile->name with string in ascending order of ASCII code. */ +int +pgFileCompareNameWithString(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + char *f2s = *(char **)f2; + + return strcmp(f1p->name, f2s); +} + +/* Compare pgFile->rel_path with string in ascending order of ASCII code. */ +int +pgFileCompareRelPathWithString(const void *f1, const void *f2) +{ + pgFile *f1p = *(pgFile **)f1; + char *f2s = *(char **)f2; + + return strcmp(f1p->rel_path, f2s); +} + /* * Compare two pgFile with their relative path and external_dir_num in ascending * order of ASСII code. @@ -478,12 +377,33 @@ pgFileCompareSize(const void *f1, const void *f2) return 0; } -static int +/* Compare two pgFile with their size in descending order */ +int +pgFileCompareSizeDesc(const void *f1, const void *f2) +{ + return -1 * pgFileCompareSize(f1, f2); +} + +int pgCompareString(const void *str1, const void *str2) { return strcmp(*(char **) str1, *(char **) str2); } +/* + * From bsearch(3): "The compar routine is expected to have two argu‐ + * ments which point to the key object and to an array member, in that order" + * But in practice this is opposite, so we took strlen from second string (search key) + * This is checked by tests.catchup.CatchupTest.test_catchup_with_exclude_path + */ +int +pgPrefixCompareString(const void *str1, const void *str2) +{ + const char *s1 = *(char **) str1; + const char *s2 = *(char **) str2; + return strncmp(s1, s2, strlen(s2)); +} + /* Compare two Oids */ int pgCompareOid(const void *f1, const void *f2) @@ -514,6 +434,8 @@ db_map_entry_free(void *entry) * * When follow_symlink is true, symbolic link is ignored and only file or * directory linked to will be listed. + * + * TODO: make it strictly local */ void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, @@ -587,7 +509,7 @@ dir_check_file(pgFile *file, bool backup_logs) pgdata_exclude_files_non_exclusive[i]) == 0) { /* Skip */ - elog(VERBOSE, "Excluding file: %s", file->name); + elog(LOG, "Excluding file: %s", file->name); return CHECK_FALSE; } } @@ -596,7 +518,7 @@ dir_check_file(pgFile *file, bool backup_logs) if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) { /* Skip */ - elog(VERBOSE, "Excluding file: %s", file->name); + elog(LOG, "Excluding file: %s", file->name); return CHECK_FALSE; } } @@ -613,10 +535,10 @@ dir_check_file(pgFile *file, bool backup_logs) */ for (i = 0; pgdata_exclude_dir[i]; i++) { - /* relative path exclude */ - if (strcmp(file->rel_path, pgdata_exclude_dir[i]) == 0) + /* exclude by dirname */ + if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) { - elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + elog(LOG, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; } } @@ -626,7 +548,7 @@ dir_check_file(pgFile *file, bool backup_logs) if (strcmp(file->rel_path, PG_LOG_DIR) == 0) { /* Skip */ - elog(VERBOSE, "Excluding directory content: %s", file->rel_path); + elog(LOG, "Excluding directory content: %s", file->rel_path); return CHECK_EXCLUDE_FALSE; } } @@ -668,26 +590,16 @@ dir_check_file(pgFile *file, bool backup_logs) */ if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) return CHECK_FALSE; - - if (sscanf_res == 3 && S_ISDIR(file->mode) && - strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0) - file->is_database = true; } else if (path_is_prefix_of_path("global", file->rel_path)) { file->tblspcOid = GLOBALTABLESPACE_OID; - - if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0) - file->is_database = true; } else if (path_is_prefix_of_path("base", file->rel_path)) { file->tblspcOid = DEFAULTTABLESPACE_OID; sscanf(file->rel_path, "base/%u/", &(file->dbOid)); - - if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0) - file->is_database = true; } /* Do not backup ptrack_init files */ @@ -703,9 +615,9 @@ dir_check_file(pgFile *file, bool backup_logs) { if (strcmp(file->name, "pg_internal.init") == 0) return CHECK_FALSE; - /* Do not backup ptrack2.x map files */ - else if (strcmp(file->name, "ptrack.map") == 0) - return CHECK_FALSE; + /* Do not backup ptrack2.x temp map files */ +// else if (strcmp(file->name, "ptrack.map") == 0) +// return CHECK_FALSE; else if (strcmp(file->name, "ptrack.map.mmap") == 0) return CHECK_FALSE; else if (strcmp(file->name, "ptrack.map.tmp") == 0) @@ -715,55 +627,10 @@ dir_check_file(pgFile *file, bool backup_logs) return CHECK_FALSE; else if (isdigit(file->name[0])) { - char *fork_name; - int len; - char suffix[MAXPGPATH]; - - fork_name = strstr(file->name, "_"); - if (fork_name) - { - /* Auxiliary fork of the relfile */ - if (strcmp(fork_name, "vm") == 0) - file->forkName = vm; - - else if (strcmp(fork_name, "fsm") == 0) - file->forkName = fsm; - - else if (strcmp(fork_name, "cfm") == 0) - file->forkName = cfm; - - else if (strcmp(fork_name, "ptrack") == 0) - file->forkName = ptrack; - - else if (strcmp(fork_name, "init") == 0) - file->forkName = init; + set_forkname(file); - /* Do not backup ptrack files */ - if (file->forkName == ptrack) - return CHECK_FALSE; - } - else - { - /* - * snapfs files: - * RELFILENODE.BLOCKNO.snapmap.SNAPID - * RELFILENODE.BLOCKNO.snap.SNAPID - */ - if (strstr(file->name, "snap") != NULL) - return true; - - len = strlen(file->name); - /* reloid.cfm */ - if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0) - return CHECK_TRUE; - - sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid), - &(file->segno), suffix); - if (sscanf_res == 0) - elog(ERROR, "Cannot parse file name \"%s\"", file->name); - else if (sscanf_res == 1 || sscanf_res == 2) - file->is_datafile = true; - } + if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */ + return CHECK_FALSE; } } @@ -774,6 +641,8 @@ dir_check_file(pgFile *file, bool backup_logs) * List files in parent->path directory. If "exclude" is true do not add into * "files" files from pgdata_exclude_files and directories from * pgdata_exclude_dir. + * + * TODO: should we check for interrupt here ? */ static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, @@ -886,7 +755,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, * * Copy of function get_tablespace_mapping() from pg_basebackup.c. */ -static const char * +const char * get_tablespace_mapping(const char *dir) { TablespaceListCell *cell; @@ -918,14 +787,14 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, for (arg_ptr = arg; *arg_ptr; arg_ptr++) { if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "directory name too long"); + elog(ERROR, "Directory name too long"); if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') ; /* skip backslash escaping = */ else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) { if (*cell->new_dir) - elog(ERROR, "multiple \"=\" signs in %s mapping\n", type); + elog(ERROR, "Multiple \"=\" signs in %s mapping\n", type); else dst = dst_ptr = cell->new_dir; } @@ -934,7 +803,7 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, } if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "invalid %s mapping format \"%s\", " + elog(ERROR, "Invalid %s mapping format \"%s\", " "must be \"OLDDIR=NEWDIR\"", type, arg); canonicalize_path(cell->old_dir); canonicalize_path(cell->new_dir); @@ -946,11 +815,11 @@ opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, * consistent with the new_dir check. */ if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "old directory is not an absolute path in %s mapping: %s\n", + elog(ERROR, "Old directory is not an absolute path in %s mapping: %s\n", type, cell->old_dir); if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "new directory is not an absolute path in %s mapping: %s\n", + elog(ERROR, "New directory is not an absolute path in %s mapping: %s\n", type, cell->new_dir); if (list->tail) @@ -991,13 +860,20 @@ opt_externaldir_map(ConfigOption *opt, const char *arg) */ void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, - bool extract_tablespaces, bool incremental, fio_location location) + bool extract_tablespaces, bool incremental, fio_location location, + const char* waldir_path) { int i; parray *links = NULL; mode_t pg_tablespace_mode = DIR_PERMISSION; char to_path[MAXPGPATH]; + if (waldir_path && !dir_is_empty(waldir_path, location)) + { + elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path); + } + + /* get tablespace map */ if (extract_tablespaces) { @@ -1062,12 +938,33 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba /* skip external directory content */ if (dir->external_dir_num != 0) continue; + /* Create WAL directory and symlink if waldir_path is setting */ + if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) { + /* get full path to PG_XLOG_DIR */ + + join_path_components(to_path, data_dir, PG_XLOG_DIR); + + elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + waldir_path, to_path); + + /* create tablespace directory from waldir_path*/ + fio_mkdir(waldir_path, pg_tablespace_mode, location); + + /* create link to linked_path */ + if (fio_symlink(waldir_path, to_path, incremental, location) < 0) + elog(ERROR, "Could not create symbolic link \"%s\": %s", + to_path, strerror(errno)); + + continue; + + + } /* tablespace_map exists */ if (links) { /* get parent dir of rel_path */ - strncpy(parent_dir, dir->rel_path, MAXPGPATH); + strlcpy(parent_dir, dir->rel_path, MAXPGPATH); get_parent_directory(parent_dir); /* check if directory is actually link to tablespace */ @@ -1084,12 +981,12 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba const char *linked_path = get_tablespace_mapping((*link)->linked); if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory is not an absolute path: %s\n", + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", linked_path); join_path_components(to_path, data_dir, dir->rel_path); - elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", + elog(LOG, "Create directory \"%s\" and symbolic link \"%s\"", linked_path, to_path); /* create tablespace directory */ @@ -1106,9 +1003,11 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba } /* This is not symlink, create directory */ - elog(VERBOSE, "Create directory \"%s\"", dir->rel_path); + elog(LOG, "Create directory \"%s\"", dir->rel_path); join_path_components(to_path, data_dir, dir->rel_path); + + // TODO check exit code fio_mkdir(to_path, dir->mode, location); } @@ -1124,7 +1023,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba * tablespace_map or tablespace_map.txt. */ void -read_tablespace_map(parray *files, const char *backup_dir) +read_tablespace_map(parray *links, const char *backup_dir) { FILE *fp; char db_path[MAXPGPATH], @@ -1134,25 +1033,27 @@ read_tablespace_map(parray *files, const char *backup_dir) join_path_components(db_path, backup_dir, DATABASE_DIR); join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); - /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path, FIO_BACKUP_HOST)) - { - elog(LOG, "there is no file tablespace_map"); - return; - } - fp = fio_open_stream(map_path, FIO_BACKUP_HOST); if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno)); + elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno)); while (fgets(buf, lengthof(buf), fp)) { - char link_name[MAXPGPATH], - path[MAXPGPATH]; - pgFile *file; + char link_name[MAXPGPATH]; + char *path; + int n = 0; + pgFile *file; + int i = 0; + + if (sscanf(buf, "%s %n", link_name, &n) != 1) + elog(ERROR, "Invalid format found in \"%s\"", map_path); + + path = buf + n; - if (sscanf(buf, "%1023s %1023s", link_name, path) != 2) - elog(ERROR, "invalid format found in \"%s\"", map_path); + /* Remove newline character at the end of string if any */ + i = strcspn(path, "\n"); + if (strlen(path) > i) + path[i] = '\0'; file = pgut_new(pgFile); memset(file, 0, sizeof(pgFile)); @@ -1162,7 +1063,7 @@ read_tablespace_map(parray *files, const char *backup_dir) file->linked = pgut_strdup(path); canonicalize_path(file->linked); - parray_append(files, file); + parray_append(links, file); } if (ferror(fp)) @@ -1179,30 +1080,49 @@ read_tablespace_map(parray *files, const char *backup_dir) * If tablespace-mapping option is supplied, all OLDDIR entries must have * entries in tablespace_map file. * - * - * TODO: maybe when running incremental restore with tablespace remapping, then - * new tablespace directory MUST be empty? because there is no way + * When running incremental restore with tablespace remapping, then + * new tablespace directory MUST be empty, because there is no way * we can be sure, that files laying there belong to our instance. + * But "force" flag allows to ignore this condition, by wiping out + * the current content on the directory. + * + * Exit codes: + * 1. backup has no tablespaces + * 2. backup has tablespaces and they are empty + * 3. backup has tablespaces and some of them are not empty */ -void -check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty) +int +check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate) { -// char this_backup_path[MAXPGPATH]; - parray *links; + parray *links = parray_new(); size_t i; TablespaceListCell *cell; pgFile *tmp_file = pgut_new(pgFile); + bool tblspaces_are_empty = true; - links = parray_new(); + elog(LOG, "Checking tablespace directories of backup %s", + backup_id_of(backup)); + + /* validate tablespace map, + * if there are no tablespaces, then there is nothing left to do + */ + if (!validate_tablespace_map(backup, no_validate)) + { + /* + * Sanity check + * If there is no tablespaces in backup, + * then using the '--tablespace-mapping' option is a mistake. + */ + if (tablespace_dirs.head != NULL) + elog(ERROR, "Backup %s has no tablespaceses, nothing to remap " + "via \"--tablespace-mapping\" option", backup_id_of(backup)); + return NoTblspc; + } -// pgBackupGetPath(backup, this_backup_path, lengthof(this_backup_path), NULL); read_tablespace_map(links, backup->root_dir); /* Sort links by the path of a linked file*/ parray_qsort(links, pgFileCompareLinked); - elog(LOG, "check tablespace directories of backup %s", - base36enc(backup->start_time)); - /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ for (cell = tablespace_dirs.head; cell; cell = cell->next) { @@ -1212,52 +1132,115 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are elog(ERROR, "--tablespace-mapping option's old directory " "doesn't have an entry in tablespace_map file: \"%s\"", cell->old_dir); - - /* For incremental restore, check that new directory is empty */ -// if (incremental) -// { -// if (!is_absolute_path(cell->new_dir)) -// elog(ERROR, "tablespace directory is not an absolute path: %s\n", -// cell->new_dir); -// -// if (!dir_is_empty(cell->new_dir, FIO_DB_HOST)) -// elog(ERROR, "restore tablespace destination is not empty: \"%s\"", -// cell->new_dir); -// } } + /* + * There is difference between incremental restore of already existing + * tablespaceses and remapped tablespaceses. + * Former are allowed to be not empty, because we treat them like an + * extension of PGDATA. + * The latter are not, unless "--force" flag is used. + * in which case the remapped directory is nuked - just to be safe, + * because it is hard to be sure that there are no some tricky corner + * cases of pages from different systems having the same crc. + * This is a strict approach. + * + * Why can`t we not nuke it and just let it roll ? + * What if user just wants to rerun failed restore with the same + * parameters? Nuking is bad for this case. + * + * Consider the example of existing PGDATA: + * .... + * pg_tablespace + * 100500-> /somedirectory + * .... + * + * We want to remap it during restore like that: + * .... + * pg_tablespace + * 100500-> /somedirectory1 + * .... + * + * Usually it is required for "/somedirectory1" to be empty, but + * in case of incremental restore with 'force' flag, which required + * of us to drop already existing content of "/somedirectory1". + * + * TODO: Ideally in case of incremental restore we must also + * drop the "/somedirectory" directory first, but currently + * we don`t do that. + */ + /* 2 - all linked directories must be empty */ for (i = 0; i < parray_num(links); i++) { pgFile *link = (pgFile *) parray_get(links, i); const char *linked_path = link->linked; - TablespaceListCell *cell; + bool remapped = false; for (cell = tablespace_dirs.head; cell; cell = cell->next) + { if (strcmp(link->linked, cell->old_dir) == 0) { linked_path = cell->new_dir; + remapped = true; break; } + } + + if (remapped) + elog(INFO, "Tablespace %s will be remapped from \"%s\" to \"%s\"", link->name, cell->old_dir, cell->new_dir); + else + elog(INFO, "Tablespace %s will be restored using old path \"%s\"", link->name, linked_path); if (!is_absolute_path(linked_path)) - elog(ERROR, "tablespace directory is not an absolute path: %s\n", + elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", linked_path); if (!dir_is_empty(linked_path, FIO_DB_HOST)) { + if (!incremental) - elog(ERROR, "restore tablespace destination is not empty: \"%s\"", - linked_path); - *tblspaces_are_empty = false; + elog(ERROR, "Restore tablespace destination is not empty: \"%s\"", linked_path); + + else if (remapped && !force) + elog(ERROR, "Remapped tablespace destination is not empty: \"%s\". " + "Use \"--force\" flag if you want to automatically clean up the " + "content of new tablespace destination", + linked_path); + + else if (pgdata_is_empty && !force) + elog(ERROR, "PGDATA is empty, but tablespace destination is not: \"%s\". " + "Use \"--force\" flag is you want to automatically clean up the " + "content of tablespace destination", + linked_path); + + /* + * TODO: compile the list of tblspc Oids to delete later, + * similar to what we do with database_map. + */ + else if (force && (pgdata_is_empty || remapped)) + { + elog(WARNING, "Cleaning up the content of %s directory: \"%s\"", + remapped ? "remapped tablespace" : "tablespace", linked_path); + cleanup_tablespace(linked_path); + continue; + } + + tblspaces_are_empty = false; } } free(tmp_file); parray_walk(links, pgFileFree); parray_free(links); + + if (tblspaces_are_empty) + return EmptyTblspc; + + return NotEmptyTblspc; } +/* TODO: Make it consistent with check_tablespace_mapping */ void check_external_dir_mapping(pgBackup *backup, bool incremental) { @@ -1266,7 +1249,7 @@ check_external_dir_mapping(pgBackup *backup, bool incremental) int i; elog(LOG, "check external directories of backup %s", - base36enc(backup->start_time)); + backup_id_of(backup)); if (!backup->external_dir_str) { @@ -1335,7 +1318,7 @@ get_external_remap(char *current_dir) return current_dir; } -/* Parsing states for get_control_value() */ +/* Parsing states for get_control_value_str() */ #define CONTROL_WAIT_NAME 1 #define CONTROL_INNAME 2 #define CONTROL_WAIT_COLON 3 @@ -1349,26 +1332,62 @@ get_external_remap(char *current_dir) * The line has the following format: * {"name1":"value1", "name2":"value2"} * - * The value will be returned to "value_str" as string if it is not NULL. If it - * is NULL the value will be returned to "value_int64" as int64. + * The value will be returned in "value_int64" as int64. + * + * Returns true if the value was found in the line and parsed. + */ +bool +get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory) +{ + + char buf_int64[32]; + + assert(value_int64); + + /* Set default value */ + *value_int64 = 0; + + if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory)) + return false; + + if (!parse_int64(buf_int64, value_int64, 0)) + { + /* We assume that too big value is -1 */ + if (errno == ERANGE) + *value_int64 = BYTES_INVALID; + else + control_string_bad_format(str); + return false; + } + + return true; +} + +/* + * Get value from json-like line "str" of backup_content.control file. + * + * The line has the following format: + * {"name1":"value1", "name2":"value2"} + * + * The value will be returned to "value_str" as string. * * Returns true if the value was found in the line. */ -static bool -get_control_value(const char *str, const char *name, - char *value_str, int64 *value_int64, bool is_mandatory) + +bool +get_control_value_str(const char *str, const char *name, + char *value_str, size_t value_str_size, bool is_mandatory) { int state = CONTROL_WAIT_NAME; char *name_ptr = (char *) name; char *buf = (char *) str; - char buf_int64[32], /* Buffer for "value_int64" */ - *buf_int64_ptr = buf_int64; + char *const value_str_start = value_str; - /* Set default values */ - if (value_str) - *value_str = '\0'; - else if (value_int64) - *value_int64 = 0; + assert(value_str); + assert(value_str_size > 0); + + /* Set default value */ + *value_str = '\0'; while (*buf) { @@ -1378,7 +1397,7 @@ get_control_value(const char *str, const char *name, if (*buf == '"') state = CONTROL_INNAME; else if (IsAlpha(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_INNAME: /* Found target field. Parse value. */ @@ -1397,57 +1416,32 @@ get_control_value(const char *str, const char *name, if (*buf == ':') state = CONTROL_WAIT_VALUE; else if (!IsSpace(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_WAIT_VALUE: if (*buf == '"') { state = CONTROL_INVALUE; - buf_int64_ptr = buf_int64; } else if (IsAlpha(*buf)) - goto bad_format; + control_string_bad_format(str); break; case CONTROL_INVALUE: /* Value was parsed, exit */ if (*buf == '"') { - if (value_str) - { - *value_str = '\0'; - } - else if (value_int64) - { - /* Length of buf_uint64 should not be greater than 31 */ - if (buf_int64_ptr - buf_int64 >= 32) - elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - - *buf_int64_ptr = '\0'; - if (!parse_int64(buf_int64, value_int64, 0)) - { - /* We assume that too big value is -1 */ - if (errno == ERANGE) - *value_int64 = BYTES_INVALID; - else - goto bad_format; - } - } - + *value_str = '\0'; return true; } else { - if (value_str) - { - *value_str = *buf; - value_str++; - } - else - { - *buf_int64_ptr = *buf; - buf_int64_ptr++; + /* verify if value_str not exceeds value_str_size limits */ + if (value_str - value_str_start >= value_str_size - 1) { + elog(ERROR, "Field \"%s\" is out of range in the line %s of the file %s", + name, str, DATABASE_FILE_LIST); } + *value_str = *buf; + value_str++; } break; case CONTROL_WAIT_NEXT_NAME: @@ -1464,135 +1458,20 @@ get_control_value(const char *str, const char *name, /* There is no close quotes */ if (state == CONTROL_INNAME || state == CONTROL_INVALUE) - goto bad_format; + control_string_bad_format(str); /* Did not find target field */ if (is_mandatory) - elog(ERROR, "field \"%s\" is not found in the line %s of the file %s", + elog(ERROR, "Field \"%s\" is not found in the line %s of the file %s", name, str, DATABASE_FILE_LIST); return false; - -bad_format: - elog(ERROR, "%s file has invalid format in line %s", - DATABASE_FILE_LIST, str); - return false; /* Make compiler happy */ } -/* - * Construct parray of pgFile from the backup content list. - * If root is not NULL, path will be absolute path. - */ -parray * -dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location, pg_crc32 expected_crc) +static void +control_string_bad_format(const char* str) { - FILE *fp; - parray *files; - char buf[BLCKSZ]; - char stdio_buf[STDIO_BUFSIZE]; - pg_crc32 content_crc = 0; - - fp = fio_open_stream(file_txt, location); - if (fp == NULL) - elog(ERROR, "cannot open \"%s\": %s", file_txt, strerror(errno)); - - /* enable stdio buffering for local file */ - if (!fio_is_remote(location)) - setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); - - files = parray_new(); - - INIT_FILE_CRC32(true, content_crc); - - while (fgets(buf, lengthof(buf), fp)) - { - char path[MAXPGPATH]; - char linked[MAXPGPATH]; - char compress_alg_string[MAXPGPATH]; - int64 write_size, - mode, /* bit length of mode_t depends on platforms */ - is_datafile, - is_cfs, - external_dir_num, - crc, - segno, - n_blocks, - n_headers, - dbOid, /* used for partial restore */ - hdr_crc, - hdr_off, - hdr_size; - pgFile *file; - - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); - - get_control_value(buf, "path", path, NULL, true); - get_control_value(buf, "size", NULL, &write_size, true); - get_control_value(buf, "mode", NULL, &mode, true); - get_control_value(buf, "is_datafile", NULL, &is_datafile, true); - get_control_value(buf, "is_cfs", NULL, &is_cfs, false); - get_control_value(buf, "crc", NULL, &crc, true); - get_control_value(buf, "compress_alg", compress_alg_string, NULL, false); - get_control_value(buf, "external_dir_num", NULL, &external_dir_num, false); - get_control_value(buf, "dbOid", NULL, &dbOid, false); - - file = pgFileInit(path); - file->write_size = (int64) write_size; - file->mode = (mode_t) mode; - file->is_datafile = is_datafile ? true : false; - file->is_cfs = is_cfs ? true : false; - file->crc = (pg_crc32) crc; - file->compress_alg = parse_compress_alg(compress_alg_string); - file->external_dir_num = external_dir_num; - file->dbOid = dbOid ? dbOid : 0; - - /* - * Optional fields - */ - - if (get_control_value(buf, "linked", linked, NULL, false) && linked[0]) - { - file->linked = pgut_strdup(linked); - canonicalize_path(file->linked); - } - - if (get_control_value(buf, "segno", NULL, &segno, false)) - file->segno = (int) segno; - - if (get_control_value(buf, "n_blocks", NULL, &n_blocks, false)) - file->n_blocks = (int) n_blocks; - - if (get_control_value(buf, "n_headers", NULL, &n_headers, false)) - file->n_headers = (int) n_headers; - - if (get_control_value(buf, "hdr_crc", NULL, &hdr_crc, false)) - file->hdr_crc = (pg_crc32) hdr_crc; - - if (get_control_value(buf, "hdr_off", NULL, &hdr_off, false)) - file->hdr_off = hdr_off; - - if (get_control_value(buf, "hdr_size", NULL, &hdr_size, false)) - file->hdr_size = (int) hdr_size; - - parray_append(files, file); - } - - FIN_FILE_CRC32(true, content_crc); - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", file_txt); - - fio_close_stream(fp); - - if (expected_crc != 0 && - expected_crc != content_crc) - { - elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", - file_txt, content_crc, expected_crc); - return NULL; - } - - return files; + elog(ERROR, "%s file has invalid format in line %s", + DATABASE_FILE_LIST, str); } /* @@ -1610,7 +1489,7 @@ dir_is_empty(const char *path, fio_location location) /* Directory in path doesn't exist */ if (errno == ENOENT) return true; - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", path, strerror(errno)); } errno = 0; @@ -1626,7 +1505,7 @@ dir_is_empty(const char *path, fio_location location) return false; } if (errno) - elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot read directory \"%s\": %s", path, strerror(errno)); fio_closedir(dir); @@ -1787,7 +1666,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ FIO_BACKUP_HOST); file->crc = pgFileGetCRC(database_map_path, true, false); file->write_size = file->size; - file->uncompressed_size = file->read_size; + file->uncompressed_size = file->size; parray_append(backup_files_list, file); } @@ -1804,7 +1683,6 @@ read_database_map(pgBackup *backup) char path[MAXPGPATH]; char database_map_path[MAXPGPATH]; -// pgBackupGetPath(backup, path, lengthof(path), DATABASE_DIR); join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); @@ -1827,8 +1705,8 @@ read_database_map(pgBackup *backup) db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); - get_control_value(buf, "dbOid", NULL, &dbOid, true); - get_control_value(buf, "datname", datname, NULL, true); + get_control_value_int64(buf, "dbOid", &dbOid, true); + get_control_value_str(buf, "datname", datname, sizeof(datname), true); db_entry->dbOid = dbOid; db_entry->datname = pgut_strdup(datname); @@ -1850,3 +1728,143 @@ read_database_map(pgBackup *backup) return database_map; } + +/* + * Use it to cleanup tablespaces + * TODO: Current algorihtm is not very efficient in remote mode, + * due to round-trip to delete every file. + */ +void +cleanup_tablespace(const char *path) +{ + int i; + char fullpath[MAXPGPATH]; + parray *files = parray_new(); + + fio_list_dir(files, path, false, false, false, false, false, 0); + + /* delete leaf node first */ + parray_qsort(files, pgFileCompareRelPathWithExternalDesc); + + for (i = 0; i < parray_num(files); i++) + { + pgFile *file = (pgFile *) parray_get(files, i); + + join_path_components(fullpath, path, file->rel_path); + + fio_delete(file->mode, fullpath, FIO_DB_HOST); + elog(LOG, "Deleted file \"%s\"", fullpath); + } + + parray_walk(files, pgFileFree); + parray_free(files); +} + +/* + * Clear the synchronisation locks in a parray of (pgFile *)'s + */ +void +pfilearray_clear_locks(parray *file_list) +{ + int i; + for (i = 0; i < parray_num(file_list); i++) + { + pgFile *file = (pgFile *) parray_get(file_list, i); + pg_atomic_clear_flag(&file->lock); + } +} + +static inline bool +is_forkname(char *name, size_t *pos, const char *forkname) +{ + size_t fnlen = strlen(forkname); + if (strncmp(name + *pos, forkname, fnlen) != 0) + return false; + *pos += fnlen; + return true; +} + +#define OIDCHARS 10 +#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1) +#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */ + +/* Set forkName if possible */ +bool +set_forkname(pgFile *file) +{ + size_t i = 0; + uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */ + uint64_t segno = 0; + + /* pretend it is not relation file */ + file->relOid = 0; + file->forkName = none; + file->is_datafile = false; + + for (i = 0; isdigit(file->name[i]); i++) + { + if (i == 0 && file->name[i] == '0') + return false; + oid = oid * 10 + file->name[i] - '0'; + } + if (i == 0 || i > OIDCHARS || oid > UINT32_MAX) + return false; + + /* usual fork name */ + /* /^\d+_(vm|fsm|init|ptrack)$/ */ + if (is_forkname(file->name, &i, "_vm")) + file->forkName = vm; + else if (is_forkname(file->name, &i, "_fsm")) + file->forkName = fsm; + else if (is_forkname(file->name, &i, "_init")) + file->forkName = init; + else if (is_forkname(file->name, &i, "_ptrack")) + file->forkName = ptrack; + + /* segment number */ + /* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */ + if (file->name[i] == '.' && isdigit(file->name[i+1])) + { + size_t start = i+1; + for (i++; isdigit(file->name[i]); i++) + { + if (i == start && file->name[i] == '0') + return false; + segno = segno * 10 + file->name[i] - '0'; + } + if (i - start > SEGNOCHARS || segno > MAXSEGNO) + return false; + } + + /* CFS family fork names */ + if (file->forkName == none && + is_forkname(file->name, &i, ".cfm.bck")) + { + /* /^\d+(\.\d+)?\.cfm\.bck$/ */ + file->forkName = cfm_bck; + } + if (file->forkName == none && + is_forkname(file->name, &i, ".bck")) + { + /* /^\d+(\.\d+)?\.bck$/ */ + file->forkName = cfs_bck; + } + if (file->forkName == none && + is_forkname(file->name, &i, ".cfm")) + { + /* /^\d+(\.\d+)?.cfm$/ */ + file->forkName = cfm; + } + + /* If there are excess characters, it is not relation file */ + if (file->name[i] != 0) + { + file->forkName = none; + return false; + } + + file->relOid = oid; + file->segno = segno; + file->is_datafile = file->forkName == none; + return true; +} \ No newline at end of file diff --git a/src/fetch.c b/src/fetch.c index bef30dac6..5401d815e 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -92,7 +92,7 @@ fetchFile(PGconn *conn, const char *filename, size_t *filesize) /* sanity check the result set */ if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) - elog(ERROR, "unexpected result set while fetching remote file \"%s\"", + elog(ERROR, "Unexpected result set while fetching remote file \"%s\"", filename); /* Read result to local variables */ diff --git a/src/help.c b/src/help.c index 2b5bcd06e..e18706a13 100644 --- a/src/help.c +++ b/src/help.c @@ -2,13 +2,16 @@ * * help.c * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ +#include #include "pg_probackup.h" +static void help_nocmd(void); +static void help_internal(void); static void help_init(void); static void help_backup(void); static void help_restore(void); @@ -24,68 +27,74 @@ static void help_del_instance(void); static void help_archive_push(void); static void help_archive_get(void); static void help_checkdb(void); +static void help_help(void); +static void help_version(void); +static void help_catchup(void); void -help_command(char *command) +help_print_version(void) { - if (strcmp(command, "init") == 0) - help_init(); - else if (strcmp(command, "backup") == 0) - help_backup(); - else if (strcmp(command, "restore") == 0) - help_restore(); - else if (strcmp(command, "validate") == 0) - help_validate(); - else if (strcmp(command, "show") == 0) - help_show(); - else if (strcmp(command, "delete") == 0) - help_delete(); - else if (strcmp(command, "merge") == 0) - help_merge(); - else if (strcmp(command, "set-backup") == 0) - help_set_backup(); - else if (strcmp(command, "set-config") == 0) - help_set_config(); - else if (strcmp(command, "show-config") == 0) - help_show_config(); - else if (strcmp(command, "add-instance") == 0) - help_add_instance(); - else if (strcmp(command, "del-instance") == 0) - help_del_instance(); - else if (strcmp(command, "archive-push") == 0) - help_archive_push(); - else if (strcmp(command, "archive-get") == 0) - help_archive_get(); - else if (strcmp(command, "checkdb") == 0) - help_checkdb(); - else if (strcmp(command, "--help") == 0 - || strcmp(command, "help") == 0 - || strcmp(command, "-?") == 0 - || strcmp(command, "--version") == 0 - || strcmp(command, "version") == 0 - || strcmp(command, "-V") == 0) - printf(_("No help page for \"%s\" command. Try pg_probackup help\n"), command); - else - printf(_("Unknown command \"%s\". Try pg_probackup help\n"), command); - exit(0); +#ifdef PGPRO_VERSION + fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, + PGPRO_VERSION, PGPRO_EDITION); +#else + fprintf(stdout, "%s %s (PostgreSQL %s)\n", + PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); +#endif +} + +void +help_command(ProbackupSubcmd const subcmd) +{ + typedef void (* help_function_ptr)(void); + /* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ + static help_function_ptr const help_functions[] = + { + &help_nocmd, + &help_init, + &help_add_instance, + &help_del_instance, + &help_archive_push, + &help_archive_get, + &help_backup, + &help_restore, + &help_validate, + &help_delete, + &help_merge, + &help_show, + &help_set_config, + &help_set_backup, + &help_show_config, + &help_checkdb, + &help_internal, // SSH_CMD + &help_internal, // AGENT_CMD + &help_help, + &help_version, + &help_catchup, + }; + + Assert((int)subcmd < sizeof(help_functions) / sizeof(help_functions[0])); + help_functions[(int)subcmd](); } void help_pg_probackup(void) { - printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n\n"), PROGRAM_NAME); + printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n"), PROGRAM_NAME); - printf(_(" %s help [COMMAND]\n"), PROGRAM_NAME); + printf(_("\n %s help [COMMAND]\n"), PROGRAM_NAME); printf(_("\n %s version\n"), PROGRAM_NAME); - printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); + printf(_("\n %s init -B backup-dir\n"), PROGRAM_NAME); - printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s set-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -105,29 +114,32 @@ help_pg_probackup(void) printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); - printf(_("\n %s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s set-backup -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--ttl=interval] [--expire-time=timestamp]\n")); printf(_(" [--note=text]\n")); printf(_(" [--help]\n")); - printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s show-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n")); + printf(_(" [--no-scale-units]\n")); printf(_(" [--help]\n")); - printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s backup -B backup-dir -b backup-mode --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name]] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); @@ -145,7 +157,7 @@ help_pg_probackup(void) printf(_(" [--help]\n")); - printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s restore -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -161,8 +173,10 @@ help_pg_probackup(void) printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs] [--no-sync]\n")); + printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include | --db-exclude]\n")); + printf(_(" [--destroy-all-other-dbs]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); @@ -170,7 +184,7 @@ help_pg_probackup(void) printf(_(" [--archive-port=port] [--archive-user=username]\n")); printf(_(" [--help]\n")); - printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n %s validate -B backup-dir [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -179,45 +193,47 @@ help_pg_probackup(void) printf(_(" [--skip-block-validation]\n")); printf(_(" [--help]\n")); - printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n %s checkdb [-B backup-dir] [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed]\n")); + printf(_(" [--heapallindexed] [--checkunique]\n")); printf(_(" [--help]\n")); - printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_("\n %s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance-name [-i backup-id]]\n")); printf(_(" [--format=format] [--archive]\n")); - printf(_(" [--help]\n")); + printf(_(" [--no-color] [--help]\n")); - printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s delete -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); printf(_(" [--wal-depth=wal-depth]\n")); printf(_(" [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n")); printf(_(" [--delete-wal]\n")); - printf(_(" [--dry-run]\n")); + printf(_(" [--dry-run] [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); - printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s merge -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [--progress] [-j num-threads]\n")); + printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--help]\n")); - printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n %s add-instance -B backup-dir -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [--external-dirs=external-directories-paths]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); - printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n %s del-instance -B backup-dir\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [--help]\n")); - printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s archive-push -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); @@ -229,7 +245,7 @@ help_pg_probackup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); - printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n %s archive-get -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); @@ -239,41 +255,69 @@ help_pg_probackup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--help]\n")); + printf(_("\n %s catchup -b catchup-mode\n"), PROGRAM_NAME); + printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); + printf(_(" --destination-pgdata=path_to_local_dir\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot | --perm-slot]]\n")); + printf(_(" [-j num-threads]\n")); + printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--exclude-path=path_prefix]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--dry-run]\n")); + printf(_(" [--help]\n")); + if ((PROGRAM_URL || PROGRAM_EMAIL)) { printf("\n"); if (PROGRAM_URL) - printf("Read the website for details. <%s>\n", PROGRAM_URL); + printf(_("Read the website for details <%s>.\n"), PROGRAM_URL); if (PROGRAM_EMAIL) - printf("Report bugs to <%s>.\n", PROGRAM_EMAIL); + printf(_("Report bugs to <%s>.\n"), PROGRAM_EMAIL); } - exit(0); +} + +static void +help_nocmd(void) +{ + printf(_("\nUnknown command. Try pg_probackup help\n\n")); +} + +static void +help_internal(void) +{ + printf(_("\nThis command is intended for internal use\n\n")); } static void help_init(void) { - printf(_("\n%s init -B backup-path\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n\n")); + printf(_("\n%s init -B backup-dir\n\n"), PROGRAM_NAME); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n\n")); } static void help_backup(void) { - printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s backup -B backup-dir -b backup-mode --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name] [--temp-slot]\n")); + printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-E external-directories-paths]\n")); printf(_(" [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); + printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); @@ -289,9 +333,9 @@ help_backup(void) printf(_(" [--ssh-options]\n")); printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); printf(_(" --stream stream the transaction log and include it in the backup\n")); @@ -316,6 +360,12 @@ help_backup(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -329,6 +379,7 @@ help_backup(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Retention options:\n")); printf(_(" --delete-expired delete backups expired according to current\n")); @@ -391,15 +442,17 @@ help_backup(void) static void help_restore(void) { - printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s restore -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); printf(_(" [--progress] [--force] [--no-sync]\n")); printf(_(" [--no-validate] [--skip-block-validation]\n")); printf(_(" [-T OLDDIR=NEWDIR]\n")); printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); printf(_(" [--skip-external-dirs]\n")); + printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); printf(_(" [--db-include dbname | --db-exclude dbname]\n")); + printf(_(" [--destroy-all-other-dbs]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); printf(_(" [--recovery-target-timeline=timeline]\n")); @@ -416,8 +469,8 @@ help_restore(void) printf(_(" [--archive-host=hostname] [--archive-port=port]\n")); printf(_(" [--archive-user=username]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -i, --backup-id=backup-id backup to restore\n")); @@ -435,6 +488,10 @@ help_restore(void) printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); printf(_(" --skip-external-dirs do not restore all external directories\n")); + + printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n")); + + printf(_("\n Incremental restore options:\n")); printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); printf(_(" reuse valid pages available in PGDATA if they have not changed\n")); @@ -443,6 +500,9 @@ help_restore(void) printf(_("\n Partial restore options:\n")); printf(_(" --db-include dbname restore only specified databases\n")); printf(_(" --db-exclude dbname do not restore specified databases\n")); + printf(_(" --destroy-all-other-dbs\n")); + printf(_(" allows to do partial restore that is prohibited by default,\n")); + printf(_(" because it might remove all other databases.\n")); printf(_("\n Recovery options:\n")); printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); @@ -476,6 +536,12 @@ help_restore(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -489,6 +555,7 @@ help_restore(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); @@ -510,7 +577,7 @@ help_restore(void) static void help_validate(void) { - printf(_("\n%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s validate -B backup-dir [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); @@ -518,8 +585,8 @@ help_validate(void) printf(_(" [--recovery-target-name=target-name]\n")); printf(_(" [--skip-block-validation]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to validate\n")); printf(_(" --progress show progress\n")); @@ -542,6 +609,12 @@ help_validate(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -554,19 +627,20 @@ help_validate(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void help_checkdb(void) { - printf(_("\n%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); + printf(_("\n%s checkdb [-B backup-dir] [--instance=instance-name]\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed]\n\n")); + printf(_(" [--heapallindexed] [--checkunique]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" --progress show progress\n")); @@ -578,6 +652,8 @@ help_checkdb(void) printf(_(" using 'amcheck' or 'amcheck_next' extensions\n")); printf(_(" --heapallindexed also check that heap is indexed\n")); printf(_(" can be used only with '--amcheck' option\n")); + printf(_(" --checkunique also check unique constraints\n")); + printf(_(" can be used only with '--amcheck' option\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -586,6 +662,12 @@ help_checkdb(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); @@ -599,6 +681,7 @@ help_checkdb(void) printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n")); printf(_("\n Connection options:\n")); printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); @@ -612,32 +695,36 @@ help_checkdb(void) static void help_show(void) { - printf(_("\n%s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); + printf(_("\n%s show -B backup-dir\n"), PROGRAM_NAME); + printf(_(" [--instance=instance-name [-i backup-id]]\n")); printf(_(" [--format=format] [--archive]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name show info about specific instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name show info about specific instance\n")); printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); printf(_(" --archive show WAL archive information\n")); - printf(_(" --format=format show format=PLAIN|JSON\n\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --no-color disable the coloring for plain format\n\n")); } static void help_delete(void) { - printf(_("\n%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s delete -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n")); printf(_(" [-j num-threads] [--progress]\n")); printf(_(" [--retention-redundancy=retention-redundancy]\n")); printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n\n")); + printf(_(" [--wal-depth=wal-depth]\n")); + printf(_(" [--no-validate] [--no-sync]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to delete\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); + printf(_(" --no-validate disable validation during retention merge\n")); + printf(_(" --no-sync do not sync merged files to disk\n")); printf(_("\n Retention options:\n")); printf(_(" --delete-expired delete backups expired according to current\n")); @@ -661,6 +748,12 @@ help_delete(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -673,28 +766,34 @@ help_delete(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void help_merge(void) { - printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s merge -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id [-j num-threads] [--progress]\n")); + printf(_(" [--no-validate] [--no-sync]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-console=log-format-console]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); printf(_(" [--log-rotation-size=log-rotation-size]\n")); printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -i, --backup-id=backup-id backup to merge\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --progress show progress\n")); + printf(_(" --no-validate disable validation during retention merge\n")); + printf(_(" --no-sync do not sync merged files to disk\n")); printf(_("\n Logging options:\n")); printf(_(" --log-level-console=log-level-console\n")); @@ -703,6 +802,12 @@ help_merge(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-console=log-format-console\n")); + printf(_(" defines the format of the console log (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -715,13 +820,14 @@ help_merge(void) printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); printf(_(" --log-rotation-age=log-rotation-age\n")); printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); } static void help_set_backup(void) { - printf(_("\n%s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s set-backup -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" -i backup-id\n")); printf(_(" [--ttl=interval] [--expire-time=time] [--note=text]\n\n")); @@ -737,12 +843,13 @@ help_set_backup(void) static void help_set_config(void) { - printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s set-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [-D pgdata-path]\n")); printf(_(" [-E external-directories-paths]\n")); printf(_(" [--restore-command=cmdline]\n")); printf(_(" [--log-level-console=log-level-console]\n")); printf(_(" [--log-level-file=log-level-file]\n")); + printf(_(" [--log-format-file=log-format-file]\n")); printf(_(" [--log-filename=log-filename]\n")); printf(_(" [--error-log-filename=error-log-filename]\n")); printf(_(" [--log-directory=log-directory]\n")); @@ -759,8 +866,8 @@ help_set_config(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); @@ -774,6 +881,9 @@ help_set_config(void) printf(_(" --log-level-file=log-level-file\n")); printf(_(" level for file logging (default: off)\n")); printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); printf(_(" --log-filename=log-filename\n")); printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); @@ -839,27 +949,28 @@ help_set_config(void) static void help_show_config(void) { - printf(_("\n%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s show-config -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" [--format=format]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" --format=format show format=PLAIN|JSON\n\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance\n")); + printf(_(" --format=format show format=PLAIN|JSON\n")); + printf(_(" --no-scale-units show memory and time values in default units\n\n")); } static void help_add_instance(void) { - printf(_("\n%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); + printf(_("\n%s add-instance -B backup-dir -D pgdata-path\n"), PROGRAM_NAME); + printf(_(" --instance=instance-name\n")); printf(_(" [-E external-directory-path]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" --instance=instance_name name of the new instance\n")); + printf(_(" --instance=instance-name name of the new instance\n")); printf(_(" -E --external-dirs=external-directories-paths\n")); printf(_(" backup some directories not from pgdata \n")); @@ -874,22 +985,47 @@ help_add_instance(void) printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); } static void help_del_instance(void) { - printf(_("\n%s del-instance -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s del-instance -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n\n")); } static void help_archive_push(void) { - printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); + printf(_("\n%s archive-push -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--archive-timeout=timeout]\n")); printf(_(" [--no-ready-rename] [--no-sync]\n")); @@ -900,10 +1036,12 @@ help_archive_push(void) printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n")); printf(_(" --wal-file-name=wal-file-name\n")); printf(_(" name of the file to copy into WAL archive\n")); + printf(_(" --wal-file-path=wal-file-path\n")); + printf(_(" relative destination path of the WAL archive\n")); printf(_(" -j, --threads=NUM number of parallel threads\n")); printf(_(" --batch-size=NUM number of files to be copied\n")); printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n")); @@ -918,6 +1056,30 @@ help_archive_push(void) printf(_(" --compress-level=compress-level\n")); printf(_(" level of compression [0-9] (default: 1)\n")); + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); @@ -933,17 +1095,17 @@ help_archive_push(void) static void help_archive_get(void) { - printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-path=wal-file-path\n")); + printf(_("\n%s archive-get -B backup-dir --instance=instance-name\n"), PROGRAM_NAME); printf(_(" --wal-file-name=wal-file-name\n")); + printf(_(" [--wal-file-path=wal-file-path]\n")); printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); printf(_(" [--no-validate-wal]\n")); printf(_(" [--remote-proto] [--remote-host]\n")); printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); printf(_(" [--ssh-options]\n\n")); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); + printf(_(" -B, --backup-path=backup-dir location of the backup storage area\n")); + printf(_(" --instance=instance-name name of the instance to delete\n")); printf(_(" --wal-file-path=wal-file-path\n")); printf(_(" relative destination path name of the WAL file on the server\n")); printf(_(" --wal-file-name=wal-file-name\n")); @@ -953,6 +1115,30 @@ help_archive_get(void) printf(_(" --prefetch-dir=path location of the store area for prefetched WAL files\n")); printf(_(" --no-validate-wal skip validation of prefetched WAL file before using it\n")); + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + printf(_("\n Remote options:\n")); printf(_(" --remote-proto=protocol remote protocol to use\n")); printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); @@ -964,3 +1150,95 @@ help_archive_get(void) printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); } + +static void +help_help(void) +{ + printf(_("\n%s help [command]\n"), PROGRAM_NAME); + printf(_("%s command --help\n\n"), PROGRAM_NAME); +} + +static void +help_version(void) +{ + printf(_("\n%s version\n"), PROGRAM_NAME); + printf(_("%s --version\n\n"), PROGRAM_NAME); +} + +static void +help_catchup(void) +{ + printf(_("\n%s catchup -b catchup-mode\n"), PROGRAM_NAME); + printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); + printf(_(" --destination-pgdata=path_to_local_dir\n")); + printf(_(" [--stream [-S slot-name]] [--temp-slot | --perm-slot]\n")); + printf(_(" [-j num-threads]\n")); + printf(_(" [-T OLDDIR=NEWDIR]\n")); + printf(_(" [--exclude-path=path_prefix]\n")); + printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); + printf(_(" [-w --no-password] [-W --password]\n")); + printf(_(" [--remote-proto] [--remote-host]\n")); + printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); + printf(_(" [--ssh-options]\n")); + printf(_(" [--dry-run]\n")); + printf(_(" [--help]\n\n")); + + printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n")); + printf(_(" --stream stream the transaction log (only supported mode)\n")); + printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); + printf(_(" --temp-slot use temporary replication slot\n")); + printf(_(" -P --perm-slot create permanent replication slot\n")); + + printf(_(" -j, --threads=NUM number of parallel threads\n")); + + printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); + printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); + printf(_(" -x, --exclude-path=path_prefix files with path_prefix (relative to pgdata) will be\n")); + printf(_(" excluded from catchup (can be used multiple times)\n")); + printf(_(" Dangerous option! Use at your own risk!\n")); + + printf(_("\n Connection options:\n")); + printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); + printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); + printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); + printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n\n")); + + printf(_("\n Logging options:\n")); + printf(_(" --log-level-console=log-level-console\n")); + printf(_(" level for console logging (default: info)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-level-file=log-level-file\n")); + printf(_(" level for file logging (default: off)\n")); + printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); + printf(_(" --log-format-file=log-format-file\n")); + printf(_(" defines the format of log files (default: plain)\n")); + printf(_(" available options: 'plain', 'json'\n")); + printf(_(" --log-filename=log-filename\n")); + printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); + printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); + printf(_(" --error-log-filename=error-log-filename\n")); + printf(_(" filename for error logging (default: none)\n")); + printf(_(" --log-directory=log-directory\n")); + printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); + printf(_(" --log-rotation-size=log-rotation-size\n")); + printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); + printf(_(" --log-rotation-age=log-rotation-age\n")); + printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); + printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); + + printf(_("\n Remote options:\n")); + printf(_(" --remote-proto=protocol remote protocol to use\n")); + printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); + printf(_(" --remote-host=hostname remote host address or hostname\n")); + printf(_(" --remote-port=port remote host port (default: 22)\n")); + printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); + printf(_(" (default: current binary path)\n")); + printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); + printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); + printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); + + printf(_(" --dry-run perform a trial run without any changes\n\n")); +} diff --git a/src/init.c b/src/init.c index 431ea3b70..837e2bad0 100644 --- a/src/init.c +++ b/src/init.c @@ -17,82 +17,78 @@ * Initialize backup catalog. */ int -do_init(void) +do_init(CatalogState *catalogState) { - char path[MAXPGPATH]; - char arclog_path_dir[MAXPGPATH]; int results; - results = pg_check_dir(backup_path); + results = pg_check_dir(catalogState->catalog_path); + if (results == 4) /* exists and not empty*/ - elog(ERROR, "backup catalog already exist and it's not empty"); + elog(ERROR, "The backup catalog already exists and is not empty"); else if (results == -1) /*trouble accessing directory*/ { int errno_tmp = errno; - elog(ERROR, "cannot open backup catalog directory \"%s\": %s", - backup_path, strerror(errno_tmp)); + elog(ERROR, "Cannot open backup catalog directory \"%s\": %s", + catalogState->catalog_path, strerror(errno_tmp)); } /* create backup catalog root directory */ - dir_create_dir(backup_path, DIR_PERMISSION); + dir_create_dir(catalogState->catalog_path, DIR_PERMISSION, false); /* create backup catalog data directory */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir_create_dir(path, DIR_PERMISSION); + dir_create_dir(catalogState->backup_subdir_path, DIR_PERMISSION, false); /* create backup catalog wal directory */ - join_path_components(arclog_path_dir, backup_path, "wal"); - dir_create_dir(arclog_path_dir, DIR_PERMISSION); + dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); - elog(INFO, "Backup catalog '%s' successfully inited", backup_path); + elog(INFO, "Backup catalog '%s' successfully initialized", catalogState->catalog_path); return 0; } int -do_add_instance(InstanceConfig *instance) +do_add_instance(InstanceState *instanceState, InstanceConfig *instance) { - char path[MAXPGPATH]; - char arclog_path_dir[MAXPGPATH]; struct stat st; + CatalogState *catalogState = instanceState->catalog_state; /* PGDATA is always required */ if (instance->pgdata == NULL) - elog(ERROR, "Required parameter not specified: PGDATA " - "(-D, --pgdata)"); + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata); + instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); /* Ensure that all root directories already exist */ - if (access(backup_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", backup_path); + /* TODO maybe call do_init() here instead of error?*/ + if (access(catalogState->catalog_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->catalog_path); - join_path_components(path, backup_path, BACKUPS_DIR); - if (access(path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", path); + if (access(catalogState->backup_subdir_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->backup_subdir_path); - join_path_components(arclog_path_dir, backup_path, "wal"); - if (access(arclog_path_dir, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", arclog_path_dir); + if (access(catalogState->wal_subdir_path, F_OK) != 0) + elog(ERROR, "Directory does not exist: '%s'", catalogState->wal_subdir_path); - if (stat(instance->backup_instance_path, &st) == 0 && S_ISDIR(st.st_mode)) + if (stat(instanceState->instance_backup_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' backup directory already exists: '%s'", - instance->name, instance->backup_instance_path); + instanceState->instance_name, instanceState->instance_backup_subdir_path); /* * Create directory for wal files of this specific instance. * Existence check is extra paranoid because if we don't have such a * directory in data dir, we shouldn't have it in wal as well. */ - if (stat(instance->arclog_path, &st) == 0 && S_ISDIR(st.st_mode)) + if (stat(instanceState->instance_wal_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'", - instance->name, instance->arclog_path); + instanceState->instance_name, instanceState->instance_wal_subdir_path); /* Create directory for data files of this specific instance */ - dir_create_dir(instance->backup_instance_path, DIR_PERMISSION); - dir_create_dir(instance->arclog_path, DIR_PERMISSION); + dir_create_dir(instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); + dir_create_dir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); /* * Write initial configuration file. @@ -124,8 +120,8 @@ do_add_instance(InstanceConfig *instance) SOURCE_DEFAULT); /* pgdata was set through command line */ - do_set_config(true); + do_set_config(instanceState, true); - elog(INFO, "Instance '%s' successfully inited", instance_name); + elog(INFO, "Instance '%s' successfully initialized", instanceState->instance_name); return 0; } diff --git a/src/merge.c b/src/merge.c index a453a073c..e8f926795 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2,7 +2,7 @@ * * merge.c: merge FULL and incremental backups * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -30,6 +30,7 @@ typedef struct bool program_version_match; bool use_bitmap; bool is_retry; + bool no_sync; /* * Return value from the thread. @@ -50,13 +51,15 @@ static void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, const char *to_root, bool use_bitmap, - bool is_retry); + bool is_retry, bool no_sync); static void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, const char *full_database_dir, - const char *full_external_prefix); + const char *full_external_prefix, bool no_sync); + +static bool is_forward_compatible(parray *parent_chain); /* * Implementation of MERGE command. @@ -66,7 +69,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, * - Remove unnecessary files, which doesn't exist in the target backup anymore */ void -do_merge(time_t backup_id) +do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync) { parray *backups; parray *merge_list = parray_new(); @@ -76,15 +79,15 @@ do_merge(time_t backup_id) int i; if (backup_id == INVALID_BACKUP_ID) - elog(ERROR, "required parameter is not specified: --backup-id"); + elog(ERROR, "Required parameter is not specified: --backup-id"); - if (instance_name == NULL) - elog(ERROR, "required parameter is not specified: --instance"); + if (instanceState == NULL) + elog(ERROR, "Required parameter is not specified: --instance"); elog(INFO, "Merge started"); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find destination backup first */ for (i = 0; i < parray_num(backups); i++) @@ -102,7 +105,7 @@ do_merge(time_t backup_id) backup->status != BACKUP_STATUS_MERGED && backup->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); dest_backup = backup; break; @@ -151,12 +154,12 @@ do_merge(time_t backup_id) full_backup = dest_backup; dest_backup = NULL; elog(INFO, "Merge target backup %s is full backup", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* sanity */ if (full_backup->status == BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), status2str(full_backup->status)); /* Case #1 */ @@ -168,7 +171,7 @@ do_merge(time_t backup_id) elog(ERROR, "Merge target is full backup and has multiple direct children, " "you must specify child backup id you want to merge with"); - elog(LOG, "Looking for closest incremental backup to merge with"); + elog(INFO, "Looking for closest incremental backup to merge with"); /* Look for closest child backup */ for (i = 0; i < parray_num(backups); i++) @@ -191,7 +194,7 @@ do_merge(time_t backup_id) if (dest_backup == NULL) elog(ERROR, "Failed to find merge candidate, " "backup %s has no valid children", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); } /* Case #2 */ @@ -220,11 +223,9 @@ do_merge(time_t backup_id) } if (!dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(ERROR, "Full backup %s has unfinished merge with missing backup %s", - tmp_backup_id, + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); } } else if (full_backup->status == BACKUP_STATUS_MERGED) @@ -250,16 +251,14 @@ do_merge(time_t backup_id) } if (!dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(WARNING, "Full backup %s has unfinished merge with missing backup %s", - tmp_backup_id, + backup_id_of(full_backup), base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); } } else elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), + backup_id_of(full_backup), status2str(full_backup->status)); } else @@ -297,7 +296,7 @@ do_merge(time_t backup_id) if (dest_backup->status == BACKUP_STATUS_MERGING || dest_backup->status == BACKUP_STATUS_DELETING) elog(WARNING, "Rerun unfinished merge for backup %s", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); /* First we should try to find parent FULL backup */ full_backup = find_parent_full_backup(dest_backup); @@ -311,7 +310,7 @@ do_merge(time_t backup_id) */ if (dest_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Failed to find parent full backup for %s", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); /* Find FULL backup that has unfinished merge with dest backup */ for (i = 0; i < parray_num(backups); i++) @@ -328,7 +327,7 @@ do_merge(time_t backup_id) if (!full_backup) elog(ERROR, "Failed to find full backup that has unfinished merge" "with backup %s, cannot rerun merge", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); if (full_backup->status == BACKUP_STATUS_MERGED) elog(WARNING, "Incremental chain is broken, try to recover unfinished merge"); @@ -338,13 +337,12 @@ do_merge(time_t backup_id) else { if ((full_backup->status == BACKUP_STATUS_MERGED || - full_backup->status == BACKUP_STATUS_MERGED) && + full_backup->status == BACKUP_STATUS_MERGING) && dest_backup->start_time != full_backup->merge_dest_backup) { - char *tmp_backup_id = base36enc_dup(full_backup->start_time); elog(ERROR, "Full backup %s has unfinished merge with backup %s", - tmp_backup_id, base36enc(full_backup->merge_dest_backup)); - pg_free(tmp_backup_id); + backup_id_of(full_backup), + base36enc(full_backup->merge_dest_backup)); } } @@ -359,7 +357,7 @@ do_merge(time_t backup_id) * having status MERGED */ if (dest_backup == NULL && full_backup->status != BACKUP_STATUS_MERGED) elog(ERROR, "Cannot run merge for full backup %s", - base36enc(full_backup->start_time)); + backup_id_of(full_backup)); /* sanity */ if (full_backup->status != BACKUP_STATUS_OK && @@ -368,7 +366,7 @@ do_merge(time_t backup_id) full_backup->status != BACKUP_STATUS_MERGED && full_backup->status != BACKUP_STATUS_MERGING) elog(ERROR, "Backup %s has status: %s", - base36enc(full_backup->start_time), status2str(full_backup->status)); + backup_id_of(full_backup), status2str(full_backup->status)); /* Form merge list */ dest_backup_tmp = dest_backup; @@ -386,7 +384,7 @@ do_merge(time_t backup_id) dest_backup_tmp->status != BACKUP_STATUS_MERGED && dest_backup_tmp->status != BACKUP_STATUS_DELETING) elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup_tmp->start_time), + backup_id_of(dest_backup_tmp), status2str(dest_backup_tmp->status)); if (dest_backup_tmp->backup_mode == BACKUP_MODE_FULL) @@ -400,12 +398,13 @@ do_merge(time_t backup_id) parray_append(merge_list, full_backup); /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true); + catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); /* do actual merge */ - merge_chain(merge_list, full_backup, dest_backup); + merge_chain(instanceState, merge_list, full_backup, dest_backup, no_validate, no_sync); - pgBackupValidate(full_backup, NULL); + if (!no_validate) + pgBackupValidate(full_backup, NULL); if (full_backup->status == BACKUP_STATUS_CORRUPT) elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); @@ -432,10 +431,11 @@ do_merge(time_t backup_id) * that chain is ok. */ void -merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) +merge_chain(InstanceState *instanceState, + parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, + bool no_validate, bool no_sync) { int i; - char *dest_backup_id; char full_external_prefix[MAXPGPATH]; char full_database_dir[MAXPGPATH]; parray *full_externals = NULL, @@ -472,26 +472,20 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) full_backup->status == BACKUP_STATUS_MERGED) { is_retry = true; - elog(INFO, "Retry failed merge of backup %s with parent chain", base36enc(dest_backup->start_time)); + elog(INFO, "Retry failed merge of backup %s with parent chain", backup_id_of(dest_backup)); } else - elog(INFO, "Merging backup %s with parent chain", base36enc(dest_backup->start_time)); + elog(INFO, "Merging backup %s with parent chain", backup_id_of(dest_backup)); /* sanity */ if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && full_backup->merge_dest_backup != dest_backup->start_time) { - char *merge_dest_backup_current = base36enc_dup(dest_backup->start_time); - char *merge_dest_backup = base36enc_dup(full_backup->merge_dest_backup); - elog(ERROR, "Cannot run merge for %s, because full backup %s has " "unfinished merge with backup %s", - merge_dest_backup_current, - base36enc(full_backup->start_time), - merge_dest_backup); - - pg_free(merge_dest_backup_current); - pg_free(merge_dest_backup); + backup_id_of(dest_backup), + backup_id_of(full_backup), + base36enc(full_backup->merge_dest_backup)); } /* @@ -512,7 +506,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) elog(ERROR, "Backup %s has been produced by pg_probackup version %s, " "but current program version is %s. Forward compatibility " "is not supported.", - base36enc(backup->start_time), + backup_id_of(backup), backup->program_version, PROGRAM_VERSION); } @@ -532,19 +526,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * If current program version differs from destination backup version, * then in-place merge is not possible. */ - if ((parse_program_version(full_backup->program_version) == - parse_program_version(dest_backup->program_version)) && - (parse_program_version(dest_backup->program_version) == - parse_program_version(PROGRAM_VERSION))) - program_version_match = true; - else - elog(WARNING, "In-place merge is disabled because of program " - "versions mismatch. Full backup version: %s, " - "destination backup version: %s, " - "current program version: %s", - full_backup->program_version, - dest_backup->program_version, - PROGRAM_VERSION); + program_version_match = is_forward_compatible(parent_chain); /* Forbid merge retry for failed merges between 2.4.0 and any * older version. Several format changes makes it impossible @@ -564,25 +546,28 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * with sole exception of FULL backup. If it has MERGING status * then it isn't valid backup until merging is finished. */ - elog(INFO, "Validate parent chain for backup %s", - base36enc(dest_backup->start_time)); - - for (i = parray_num(parent_chain) - 1; i >= 0; i--) + if (!no_validate) { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + elog(INFO, "Validate parent chain for backup %s", + backup_id_of(dest_backup)); - /* FULL backup is not to be validated if its status is MERGING */ - if (backup->backup_mode == BACKUP_MODE_FULL && - backup->status == BACKUP_STATUS_MERGING) + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { - continue; - } + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - pgBackupValidate(backup, NULL); + /* FULL backup is not to be validated if its status is MERGING */ + if (backup->backup_mode == BACKUP_MODE_FULL && + backup->status == BACKUP_STATUS_MERGING) + { + continue; + } - if (backup->status != BACKUP_STATUS_OK) - elog(ERROR, "Backup %s has status %s, merge is aborted", - base36enc(backup->start_time), status2str(backup->status)); + pgBackupValidate(backup, NULL); + + if (backup->status != BACKUP_STATUS_OK) + elog(ERROR, "Backup %s has status %s, merge is aborted", + backup_id_of(backup), status2str(backup->status)); + } } /* @@ -607,7 +592,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) write_backup(backup, true); } else - write_backup_status(backup, BACKUP_STATUS_MERGING, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_MERGING, true); } /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ @@ -617,7 +602,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* Create directories */ create_data_directories(dest_backup->files, full_database_dir, - dest_backup->root_dir, false, false, FIO_BACKUP_HOST); + dest_backup->root_dir, false, false, FIO_BACKUP_HOST, NULL); /* External directories stuff */ if (dest_backup->external_dir_str) @@ -649,7 +634,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) makeExternalDirPathByNum(new_container, full_external_prefix, file->external_dir_num); join_path_components(dirpath, new_container, file->rel_path); - dir_create_dir(dirpath, DIR_PERMISSION); + dir_create_dir(dirpath, DIR_PERMISSION, false); } pg_atomic_init_flag(&file->lock); @@ -675,6 +660,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) arg->program_version_match = program_version_match; arg->use_bitmap = use_bitmap; arg->is_retry = is_retry; + arg->no_sync = no_sync; /* By default there are some error */ arg->ret = 1; @@ -738,7 +724,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) * We cannot set backup status to OK just yet, * because it still has old start_time. */ - StrNCpy(full_backup->program_version, PROGRAM_VERSION, + strlcpy(full_backup->program_version, PROGRAM_VERSION, sizeof(full_backup->program_version)); full_backup->parent_backup = INVALID_BACKUP_ID; full_backup->start_lsn = dest_backup->start_lsn; @@ -812,7 +798,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) join_path_components(full_file_path, full_database_dir, full_file->rel_path); pgFileDelete(full_file->mode, full_file_path); - elog(VERBOSE, "Deleted \"%s\"", full_file_path); + elog(LOG, "Deleted \"%s\"", full_file_path); } } @@ -856,13 +842,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) else { /* Ugly */ - char backups_dir[MAXPGPATH]; - char instance_dir[MAXPGPATH]; char destination_path[MAXPGPATH]; - join_path_components(backups_dir, backup_path, BACKUPS_DIR); - join_path_components(instance_dir, backups_dir, instance_name); - join_path_components(destination_path, instance_dir, + join_path_components(destination_path, instanceState->instance_backup_subdir_path, base36enc(full_backup->merge_dest_backup)); elog(LOG, "Rename %s to %s", full_backup->root_dir, destination_path); @@ -886,25 +868,26 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup) /* * Merging finished, now we can safely update ID of the FULL backup */ - dest_backup_id = base36enc_dup(full_backup->merge_dest_backup); elog(INFO, "Rename merged full backup %s to %s", - base36enc(full_backup->start_time), dest_backup_id); + backup_id_of(full_backup), + base36enc(full_backup->merge_dest_backup)); full_backup->status = BACKUP_STATUS_OK; full_backup->start_time = full_backup->merge_dest_backup; + /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ + full_backup->backup_id = full_backup->start_time; full_backup->merge_dest_backup = INVALID_BACKUP_ID; write_backup(full_backup, true); /* Critical section end */ /* Cleanup */ - pg_free(dest_backup_id); if (threads) { pfree(threads_args); pfree(threads); } - if (result_filelist && parray_num(result_filelist) > 0) + if (result_filelist) { parray_walk(result_filelist, pgFileFree); parray_free(result_filelist); @@ -962,9 +945,8 @@ merge_files(void *arg) if (S_ISDIR(dest_file->mode)) goto done; - if (progress) - elog(INFO, "Progress: (%d/%lu). Merging file \"%s\"", - i + 1, n_files, dest_file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%lu). Merging file \"%s\"", + i + 1, n_files, dest_file->rel_path); if (dest_file->is_datafile && !dest_file->is_cfs) tmp_file->segno = dest_file->segno; @@ -1069,7 +1051,7 @@ merge_files(void *arg) { BackupPageHeader2 *headers = NULL; - elog(VERBOSE, "The file didn`t changed since FULL backup, skip merge: \"%s\"", + elog(LOG, "The file didn`t changed since FULL backup, skip merge: \"%s\"", file->rel_path); tmp_file->crc = file->crc; @@ -1085,7 +1067,7 @@ merge_files(void *arg) tmp_file->hdr_crc = file->hdr_crc; } else - tmp_file->uncompressed_size = tmp_file->write_size; + tmp_file->uncompressed_size = file->uncompressed_size; /* Copy header metadata from old map into a new one */ tmp_file->n_headers = file->n_headers; @@ -1112,14 +1094,16 @@ merge_files(void *arg) dest_file, tmp_file, arguments->full_database_dir, arguments->use_bitmap, - arguments->is_retry); + arguments->is_retry, + arguments->no_sync); else merge_non_data_file(arguments->parent_chain, arguments->full_backup, arguments->dest_backup, dest_file, tmp_file, arguments->full_database_dir, - arguments->full_external_prefix); + arguments->full_external_prefix, + arguments->no_sync); done: parray_append(arguments->merge_filelist, tmp_file); @@ -1148,7 +1132,7 @@ remove_dir_with_files(const char *path) join_path_components(full_path, path, file->rel_path); pgFileDelete(file->mode, full_path); - elog(VERBOSE, "Deleted \"%s\"", full_path); + elog(LOG, "Deleted \"%s\"", full_path); } /* cleanup */ @@ -1197,7 +1181,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, char new_path[MAXPGPATH]; makeExternalDirPathByNum(old_path, externaldir_template, i + 1); makeExternalDirPathByNum(new_path, externaldir_template, from_num); - elog(VERBOSE, "Rename %s to %s", old_path, new_path); + elog(LOG, "Rename %s to %s", old_path, new_path); if (rename (old_path, new_path) == -1) elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", old_path, new_path, strerror(errno)); @@ -1212,7 +1196,8 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external, void merge_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, bool use_bitmap, bool is_retry) + const char *full_database_dir, bool use_bitmap, bool is_retry, + bool no_sync) { FILE *out = NULL; char *buffer = pgut_malloc(STDIO_BUFSIZE); @@ -1256,10 +1241,10 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, * 2 backups of old versions, where n_blocks is missing. */ - backup_data_file(NULL, tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, + backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, InvalidXLogRecPtr, BACKUP_MODE_FULL, dest_backup->compress_alg, dest_backup->compress_level, - dest_backup->checksum_version, 0, NULL, + dest_backup->checksum_version, &(full_backup->hdr_map), true); /* drop restored temp file */ @@ -1283,7 +1268,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, return; /* sync second temp file to disk */ - if (fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); @@ -1304,7 +1289,8 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, void merge_non_data_file(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, const char *to_external_prefix) + const char *full_database_dir, const char *to_external_prefix, + bool no_sync) { int i; char to_fullpath[MAXPGPATH]; @@ -1348,8 +1334,8 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, */ if (!from_file) { - elog(ERROR, "Failed to locate nonedata file \"%s\" in backup %s", - dest_file->rel_path, base36enc(from_backup->start_time)); + elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", + dest_file->rel_path, backup_id_of(from_backup)); continue; } @@ -1359,11 +1345,11 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, /* sanity */ if (!from_backup) - elog(ERROR, "Failed to found a backup containing full copy of nonedata file \"%s\"", + elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", dest_file->rel_path); if (!from_file) - elog(ERROR, "Failed to locate a full copy of nonedata file \"%s\"", dest_file->rel_path); + elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", dest_file->rel_path); /* set path to source file */ if (from_file->external_dir_num) @@ -1388,7 +1374,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); /* sync temp file to disk */ - if (fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp, strerror(errno)); @@ -1398,3 +1384,58 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, to_fullpath_tmp, to_fullpath, strerror(errno)); } + +/* + * If file format in incremental chain is compatible + * with current storage format. + * If not, then in-place merge is not possible. + * + * Consider the following examples: + * STORAGE_FORMAT_VERSION = 2.4.4 + * 2.3.3 \ + * 2.3.4 \ disable in-place merge, because + * 2.4.1 / current STORAGE_FORMAT_VERSION > 2.3.3 + * 2.4.3 / + * + * 2.4.4 \ enable in_place merge, because + * 2.4.5 / current STORAGE_FORMAT_VERSION == 2.4.4 + * + * 2.4.5 \ enable in_place merge, because + * 2.4.6 / current STORAGE_FORMAT_VERSION < 2.4.5 + * + */ +bool +is_forward_compatible(parray *parent_chain) +{ + int i; + pgBackup *oldest_ver_backup = NULL; + uint32 oldest_ver_in_chain = parse_program_version(PROGRAM_VERSION); + + for (i = 0; i < parray_num(parent_chain); i++) + { + pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); + uint32 current_version = parse_program_version(backup->program_version); + + if (!oldest_ver_backup) + oldest_ver_backup = backup; + + if (current_version < oldest_ver_in_chain) + { + oldest_ver_in_chain = current_version; + oldest_ver_backup = backup; + } + } + + if (oldest_ver_in_chain < parse_program_version(STORAGE_FORMAT_VERSION)) + { + elog(WARNING, "In-place merge is disabled because of storage format incompatibility. " + "Backup %s storage format version: %s, " + "current storage format version: %s", + backup_id_of(oldest_ver_backup), + oldest_ver_backup->program_version, + STORAGE_FORMAT_VERSION); + return false; + } + + return true; +} diff --git a/src/parsexlog.c b/src/parsexlog.c index 5a33d3045..7df169fbf 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -29,7 +29,10 @@ * RmgrNames is an array of resource manager names, to make error messages * a bit nicer. */ -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 +#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ + name, +#elif PG_VERSION_NUM >= 100000 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ name, #else @@ -148,10 +151,16 @@ typedef struct int ret; } xlog_thread_arg; +static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg); +static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data); + static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI); + int reqLen, XLogRecPtr targetRecPtr, char *readBuf +#if PG_VERSION_NUM < 130000 + ,TimeLineID *pageTLI +#endif + ); static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, TimeLineID tli, uint32 segment_size, @@ -379,11 +388,11 @@ validate_backup_wal_from_start_to_stop(pgBackup *backup, * If we don't have WAL between start_lsn and stop_lsn, * the backup is definitely corrupted. Update its status. */ - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); elog(WARNING, "There are not enough WAL records to consistenly restore " "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", - base36enc(backup->start_time), + backup_id_of(backup), (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), (uint32) (backup->stop_lsn >> 32), @@ -401,24 +410,20 @@ validate_wal(pgBackup *backup, const char *archivedir, time_t target_time, TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, uint32 wal_seg_size) { - const char *backup_id; XLogRecTarget last_rec; char last_timestamp[100], target_timestamp[100]; bool all_wal = false; - /* We need free() this later */ - backup_id = base36enc(backup->start_time); - if (!XRecOffIsValid(backup->start_lsn)) elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), - backup_id); + backup_id_of(backup)); if (!XRecOffIsValid(backup->stop_lsn)) elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), - backup_id); + backup_id_of(backup)); /* * Check that the backup has all wal files needed @@ -441,7 +446,7 @@ validate_wal(pgBackup *backup, const char *archivedir, if (backup->status == BACKUP_STATUS_CORRUPT) { - elog(WARNING, "Backup %s WAL segments are corrupted", backup_id); + elog(WARNING, "Backup %s WAL segments are corrupted", backup_id_of(backup)); return; } /* @@ -452,7 +457,7 @@ validate_wal(pgBackup *backup, const char *archivedir, !XRecOffIsValid(target_lsn)) { /* Recovery target is not given so exit */ - elog(INFO, "Backup %s WAL segments are valid", backup_id); + elog(INFO, "Backup %s WAL segments are valid", backup_id_of(backup)); return; } @@ -474,7 +479,7 @@ validate_wal(pgBackup *backup, const char *archivedir, last_rec.rec_xid = backup->recovery_xid; last_rec.rec_lsn = backup->stop_lsn; - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time); + time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, false); if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) || (target_time != 0 && backup->recovery_time >= target_time) @@ -487,7 +492,7 @@ validate_wal(pgBackup *backup, const char *archivedir, InvalidXLogRecPtr, true, validateXLogRecord, &last_rec, true); if (last_rec.rec_time > 0) time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_rec.rec_time)); + timestamptz_to_time_t(last_rec.rec_time), false); /* There are all needed WAL records */ if (all_wal) @@ -502,7 +507,7 @@ validate_wal(pgBackup *backup, const char *archivedir, (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), target_time); + time2iso(target_timestamp, lengthof(target_timestamp), target_time, false); if (TransactionIdIsValid(target_xid) && target_time != 0) elog(ERROR, "Not enough WAL records to time %s and xid " XID_FMT, target_timestamp, target_xid); @@ -551,7 +556,13 @@ read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, TimestampTz last_time = 0; char *errormsg; - record = XLogReadRecord(xlogreader, startpoint, &errormsg); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; @@ -615,7 +626,13 @@ wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, xlogreader->system_identifier = instance_config.system_identifier; - res = XLogReadRecord(xlogreader, target_lsn, &errormsg) != NULL; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(target_lsn)) + target_lsn = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, target_lsn); +#endif + + res = WalReadRecord(xlogreader, target_lsn, &errormsg) != NULL; /* Didn't find 'target_lsn' and there is no error, return false */ if (errormsg) @@ -656,6 +673,12 @@ get_first_record_lsn(const char *archivedir, XLogSegNo segno, /* Set startpoint to 0 in segno */ GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + while (attempts <= timeout) { record = XLogFindNextRecord(xlogreader, startpoint); @@ -710,6 +733,12 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, /* Set startpoint to 0 in segno */ GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -733,7 +762,7 @@ get_next_record_lsn(const char *archivedir, XLogSegNo segno, if (interrupted) elog(ERROR, "Interrupted during WAL reading"); - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { @@ -816,12 +845,26 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, */ GetXLogSegNo(start_lsn, start_segno, wal_seg_size); if (start_segno == segno) + { startpoint = start_lsn; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + } else { XLogRecPtr found; GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); + +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(startpoint)) + startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, startpoint); +#endif + found = XLogFindNextRecord(xlogreader, startpoint); if (XLogRecPtrIsInvalid(found)) @@ -846,7 +889,7 @@ get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, if (interrupted) elog(ERROR, "Interrupted during WAL reading"); - record = XLogReadRecord(xlogreader, startpoint, &errormsg); + record = WalReadRecord(xlogreader, startpoint, &errormsg); if (record == NULL) { XLogRecPtr errptr; @@ -905,8 +948,11 @@ get_gz_error(gzFile gzf) /* XLogreader callback function, to read a WAL page */ static int SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf, - TimeLineID *pageTLI) + int reqLen, XLogRecPtr targetRecPtr, char *readBuf +#if PG_VERSION_NUM < 130000 + ,TimeLineID *pageTLI +#endif + ) { XLogReaderData *reader_data; uint32 targetPageOff; @@ -970,7 +1016,7 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); - snprintf(reader_data->xlogpath, MAXPGPATH, "%s/%s", wal_archivedir, xlogfname); + join_path_components(reader_data->xlogpath, wal_archivedir, xlogfname); snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath); /* We fall back to using .partial segment in case if we are running @@ -1040,7 +1086,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, reader_data->prev_page_off == targetPageOff) { memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); +#if PG_VERSION_NUM < 130000 *pageTLI = reader_data->tli; +#endif return XLOG_BLCKSZ; } @@ -1084,7 +1132,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); reader_data->prev_page_off = targetPageOff; +#if PG_VERSION_NUM < 130000 *pageTLI = reader_data->tli; +#endif return XLOG_BLCKSZ; } @@ -1109,12 +1159,7 @@ InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, if (allocate_reader) { -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, - reader_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); -#endif + xlogreader = WalReaderAllocate(wal_seg_size, reader_data); if (xlogreader == NULL) elog(ERROR, "Out of memory"); xlogreader->system_identifier = instance_config.system_identifier; @@ -1244,6 +1289,11 @@ RunXLogThreads(const char *archivedir, time_t target_time, if (thread_args[i].ret == 1) result = false; } + thread_interrupted = false; + +// TODO: we must detect difference between actual error (failed to read WAL) and interrupt signal +// if (interrupted) +// elog(ERROR, "Interrupted during WAL parsing"); /* Release threads here, use thread_args only below */ pfree(threads); @@ -1314,16 +1364,18 @@ XLogThreadWorker(void *arg) uint32 prev_page_off = 0; bool need_read = true; -#if PG_VERSION_NUM >= 110000 - xlogreader = XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, - reader_data); -#else - xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, reader_data); -#endif + xlogreader = WalReaderAllocate(wal_seg_size, reader_data); + if (xlogreader == NULL) elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); xlogreader->system_identifier = instance_config.system_identifier; +#if PG_VERSION_NUM >= 130000 + if (XLogRecPtrIsInvalid(thread_arg->startpoint)) + thread_arg->startpoint = SizeOfXLogShortPHD; + XLogBeginRead(xlogreader, thread_arg->startpoint); +#endif + found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); /* @@ -1376,7 +1428,7 @@ XLogThreadWorker(void *arg) !SwitchThreadToNextWal(xlogreader, thread_arg)) break; - record = XLogReadRecord(xlogreader, thread_arg->startpoint, &errormsg); + record = WalReadRecord(xlogreader, thread_arg->startpoint, &errormsg); if (record == NULL) { @@ -1387,7 +1439,18 @@ XLogThreadWorker(void *arg) * Usually SimpleXLogPageRead() does it by itself. But here we need * to do it manually to support threads. */ +#if PG_VERSION_NUM >= 150000 + if (reader_data->need_switch && ( + errormsg == NULL || + /* + * Pg15 now informs if "contrecord" is missing. + * TODO: probably we should abort reading logs at this moment. + * But we continue as we did with bug present in Pg < 15. + */ + !XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))) +#else if (reader_data->need_switch && errormsg == NULL) +#endif { if (SwitchThreadToNextWal(xlogreader, thread_arg)) continue; @@ -1525,9 +1588,14 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) reader_data = (XLogReaderData *) xlogreader->private_data; reader_data->need_switch = false; +start: /* Critical section */ pthread_lock(&wal_segment_mutex); Assert(segno_next); + + if (reader_data->xlogsegno > segno_next) + segno_next = reader_data->xlogsegno; + reader_data->xlogsegno = segno_next; segnum_read++; segno_next++; @@ -1541,6 +1609,7 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) GetXLogRecPtr(reader_data->xlogsegno, 0, wal_seg_size, arg->startpoint); /* We need to close previously opened file if it wasn't closed earlier */ CleanupXLogPageRead(xlogreader); + xlogreader->currRecPtr = InvalidXLogRecPtr; /* Skip over the page header and contrecord if any */ found = XLogFindNextRecord(xlogreader, arg->startpoint); @@ -1550,6 +1619,8 @@ SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) */ if (XLogRecPtrIsInvalid(found)) { + if (reader_data->need_switch) + goto start; /* * Check if we need to stop reading. We stop if other thread found a * target segment. @@ -1716,7 +1787,12 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, /* Is this a special record type that I recognize? */ - if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE) + if (rmid == RM_DBASE_ID +#if PG_VERSION_NUM >= 150000 + && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) +#else + && rminfo == XLOG_DBASE_CREATE) +#endif { /* * New databases can be safely ignored. They would be completely @@ -1745,6 +1821,18 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, * source system. */ } + else if (rmid == RM_XACT_ID && + ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT || + (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED)) + { + /* + * These records can include "dropped rels". We can safely ignore + * them, we will see that they are missing and copy them from the + * source. + */ + } else if (info & XLR_SPECIAL_REL_UPDATE) { /* @@ -1758,13 +1846,21 @@ extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, RmgrNames[rmid], info); } +#if PG_VERSION_NUM >= 150000 + for (block_id = 0; block_id <= record->record->max_block_id; block_id++) +#else for (block_id = 0; block_id <= record->max_block_id; block_id++) +#endif { RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; +#if PG_VERSION_NUM >= 150000 + if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) +#else if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) +#endif continue; /* We only care about the main fork; others are copied as is */ @@ -1857,3 +1953,28 @@ bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_ return rc; } +static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg) +{ + +#if PG_VERSION_NUM >= 130000 + return XLogReadRecord(xlogreader, errormsg); +#else + return XLogReadRecord(xlogreader, startpoint, errormsg); +#endif + +} + +static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data) +{ + +#if PG_VERSION_NUM >= 130000 + return XLogReaderAllocate(wal_seg_size, NULL, + XL_ROUTINE(.page_read = &SimpleXLogPageRead), + reader_data); +#elif PG_VERSION_NUM >= 110000 + return XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, + reader_data); +#else + return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); +#endif +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index f2aca75fd..fa67ddff5 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -2,13 +2,46 @@ * * pg_probackup.c: Backup/Recovery manager for PostgreSQL. * + * This is an entry point for the program. + * Parse command name and it's options, verify them and call a + * do_***() function that implements the command. + * + * Avoid using global variables in the code. + * Pass all needed information as funciton arguments: + * + + * + * TODO (see pg_probackup_state.h): + * + * Functions that work with a backup catalog accept catalogState, + * which currently only contains pathes to backup catalog subdirectories + * + function specific options. + * + * Functions that work with an instance accept instanceState argument, which + * includes catalogState, instance_name, + * info about pgdata associated with the instance (see pgState), + * various instance config options, and list of backups belonging to the instance. + * + function specific options. + * + * Functions that work with multiple backups in the catalog + * accept instanceState and info needed to determine the range of backups to handle. + * + function specific options. + * + * Functions that work with a single backup accept backupState argument, + * which includes link to the instanceState, backup_id and backup-specific info. + * + function specific options. + * + * Functions that work with a postgreSQL instance (i.e. checkdb) accept pgState, + * which includes info about pgdata directory and connection. + * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2021, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" +#include "pg_probackup_state.h" #include "pg_getopt.h" #include "streamutil.h" @@ -27,67 +60,45 @@ const char *PROGRAM_FULL_PATH = NULL; const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; -typedef enum ProbackupSubcmd -{ - NO_CMD = 0, - INIT_CMD, - ADD_INSTANCE_CMD, - DELETE_INSTANCE_CMD, - ARCHIVE_PUSH_CMD, - ARCHIVE_GET_CMD, - BACKUP_CMD, - RESTORE_CMD, - VALIDATE_CMD, - DELETE_CMD, - MERGE_CMD, - SHOW_CMD, - SET_CONFIG_CMD, - SET_BACKUP_CMD, - SHOW_CONFIG_CMD, - CHECKDB_CMD -} ProbackupSubcmd; - - +/* ================ catalogState =========== */ /* directory options */ -char *backup_path = NULL; -/* - * path or to the data files in the backup catalog - * $BACKUP_PATH/backups/instance_name - */ -char backup_instance_path[MAXPGPATH]; -/* - * path or to the wal files in the backup catalog - * $BACKUP_PATH/wal/instance_name - */ -char arclog_path[MAXPGPATH] = ""; +/* TODO make it local variable, pass as an argument to all commands that need it. */ +static char *backup_path = NULL; + +static CatalogState *catalogState = NULL; +/* ================ catalogState (END) =========== */ -/* colon separated external directories list ("/path1:/path2") */ -char *externaldir = NULL; /* common options */ -static char *backup_id_string = NULL; int num_threads = 1; bool stream_wal = false; +bool no_color = false; +bool show_color = true; bool is_archive_cmd = false; pid_t my_pid = 0; __thread int my_thread_num = 1; bool progress = false; bool no_sync = false; +time_t start_time = INVALID_BACKUP_ID; #if PG_VERSION_NUM >= 100000 char *replication_slot = NULL; -#endif bool temp_slot = false; +#endif +bool perm_slot = false; /* backup options */ bool backup_logs = false; bool smooth_checkpoint; -char *remote_agent; +bool remote_agent = false; static char *backup_note = NULL; +/* catchup options */ +static char *catchup_source_pgdata = NULL; +static char *catchup_destination_pgdata = NULL; /* restore options */ static char *target_time = NULL; static char *target_xid = NULL; static char *target_lsn = NULL; static char *target_inclusive = NULL; -static TimeLineID target_tli; +static char *target_tli_string; /* timeline number, "current" or "latest"*/ static char *target_stop; static bool target_immediate; static char *target_name = NULL; @@ -99,7 +110,7 @@ static pgRecoveryTarget *recovery_target_options = NULL; static pgRestoreParams *restore_params = NULL; time_t current_time = 0; -bool restore_as_replica = false; +static bool restore_as_replica = false; bool no_validate = false; IncrRestoreMode incremental_mode = INCR_NONE; @@ -109,10 +120,16 @@ bool skip_external_dirs = false; /* array for datnames, provided via db-include and db-exclude */ static parray *datname_exclude_list = NULL; static parray *datname_include_list = NULL; +/* arrays for --exclude-path's */ +static parray *exclude_absolute_paths_list = NULL; +static parray *exclude_relative_paths_list = NULL; +static char* gl_waldir_path = NULL; +static bool allow_partial_incremental = false; /* checkdb options */ bool need_amcheck = false; bool heapallindexed = false; +bool checkunique = false; bool amcheck_parent = false; /* delete options */ @@ -123,10 +140,14 @@ bool force = false; bool dry_run = false; static char *delete_status = NULL; /* compression options */ -bool compress_shortcut = false; +static bool compress_shortcut = false; -/* other options */ -char *instance_name; +/* ================ instanceState =========== */ +static char *instance_name; + +static InstanceState *instanceState = NULL; + +/* ================ instanceState (END) =========== */ /* archive push options */ int batch_size = 1; @@ -134,6 +155,7 @@ static char *wal_file_path; static char *wal_file_name; static bool file_overwrite = false; static bool no_ready_rename = false; +static char archive_push_xlog_dir[MAXPGPATH] = ""; /* archive get options */ static char *prefetch_dir; @@ -142,15 +164,17 @@ bool no_validate_wal = false; /* show options */ ShowFormat show_format = SHOW_PLAIN; bool show_archive = false; +static bool show_base_units = false; /* set-backup options */ int64 ttl = -1; static char *expire_time_string = NULL; static pgSetBackupParams *set_backup_params = NULL; -/* current settings */ +/* ================ backupState =========== */ +static char *backup_id_string = NULL; pgBackup current; -static ProbackupSubcmd backup_subcmd = NO_CMD; +/* ================ backupState (END) =========== */ static bool help_opt = false; @@ -158,10 +182,11 @@ static void opt_incr_restore_mode(ConfigOption *opt, const char *arg); static void opt_backup_mode(ConfigOption *opt, const char *arg); static void opt_show_format(ConfigOption *opt, const char *arg); -static void compress_init(void); +static void compress_init(ProbackupSubcmd const subcmd); static void opt_datname_exclude_list(ConfigOption *opt, const char *arg); static void opt_datname_include_list(ConfigOption *opt, const char *arg); +static void opt_exclude_path(ConfigOption *opt, const char *arg); /* * Short name should be non-printable ASCII character. @@ -178,23 +203,32 @@ static ConfigOption cmd_options[] = { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, + { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, /* backup options */ { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, +#if PG_VERSION_NUM >= 100000 { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, +#endif + { 'b', 'P', "perm-slot", &perm_slot, SOURCE_CMD_STRICT }, { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, + { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, + /* catchup options */ + { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, + { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, + { 'f', 'x', "exclude-path", opt_exclude_path, SOURCE_CMD_STRICT }, /* restore options */ { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 144, "recovery-target-lsn", &target_lsn, SOURCE_CMD_STRICT }, { 's', 138, "recovery-target-inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "recovery-target-timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 139, "recovery-target-timeline", &target_tli_string, SOURCE_CMD_STRICT }, { 's', 157, "recovery-target", &target_stop, SOURCE_CMD_STRICT }, { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, @@ -209,9 +243,12 @@ static ConfigOption cmd_options[] = { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, + { 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT }, + { 'b', 242, "destroy-all-other-dbs", &allow_partial_incremental, SOURCE_CMD_STRICT }, /* checkdb options */ { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, + { 'b', 198, "checkunique", &checkunique, SOURCE_CMD_STRICT }, { 'b', 197, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, /* delete options */ { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, @@ -239,6 +276,8 @@ static ConfigOption cmd_options[] = /* show options */ { 'f', 165, "format", opt_show_format, SOURCE_CMD_STRICT }, { 'b', 166, "archive", &show_archive, SOURCE_CMD_STRICT }, + /* show-config options */ + { 'b', 167, "no-scale-units", &show_base_units,SOURCE_CMD_STRICT }, /* set-backup options */ { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, @@ -249,51 +288,42 @@ static ConfigOption cmd_options[] = { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, + { 's', 139, "timeline", &target_tli_string, SOURCE_CMD_STRICT }, { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, { 0 } }; -static void -setMyLocation(void) -{ - -#ifdef WIN32 - if (IsSshProtocol()) - elog(ERROR, "Currently remote operations on Windows are not supported"); -#endif - - MyLocation = IsSshProtocol() - ? (backup_subcmd == ARCHIVE_PUSH_CMD || backup_subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST - : (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == ADD_INSTANCE_CMD) - ? FIO_BACKUP_HOST - : FIO_LOCAL_HOST - : FIO_LOCAL_HOST; -} - /* * Entry point of pg_probackup command. */ int main(int argc, char *argv[]) { - char *command = NULL, - *command_name; + char *command = NULL; + ProbackupSubcmd backup_subcmd = NO_CMD; PROGRAM_NAME_FULL = argv[0]; + /* Check terminal presense and initialize ANSI escape codes for Windows */ + init_console(); + /* Initialize current backup */ pgBackupInit(¤t); /* Initialize current instance configuration */ + //TODO get git of this global variable craziness init_config(&instance_config, instance_name); PROGRAM_NAME = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_probackup")); PROGRAM_FULL_PATH = palloc0(MAXPGPATH); + // Setting C locale for numeric values in order to impose dot-based floating-point representation + memorize_environment_locale(); + setlocale(LC_NUMERIC, "C"); + /* Get current time */ current_time = time(NULL); @@ -316,91 +346,60 @@ main(int argc, char *argv[]) /* Parse subcommands and non-subcommand options */ if (argc > 1) { - if (strcmp(argv[1], "archive-push") == 0) - backup_subcmd = ARCHIVE_PUSH_CMD; - else if (strcmp(argv[1], "archive-get") == 0) - backup_subcmd = ARCHIVE_GET_CMD; - else if (strcmp(argv[1], "add-instance") == 0) - backup_subcmd = ADD_INSTANCE_CMD; - else if (strcmp(argv[1], "del-instance") == 0) - backup_subcmd = DELETE_INSTANCE_CMD; - else if (strcmp(argv[1], "init") == 0) - backup_subcmd = INIT_CMD; - else if (strcmp(argv[1], "backup") == 0) - backup_subcmd = BACKUP_CMD; - else if (strcmp(argv[1], "restore") == 0) - backup_subcmd = RESTORE_CMD; - else if (strcmp(argv[1], "validate") == 0) - backup_subcmd = VALIDATE_CMD; - else if (strcmp(argv[1], "delete") == 0) - backup_subcmd = DELETE_CMD; - else if (strcmp(argv[1], "merge") == 0) - backup_subcmd = MERGE_CMD; - else if (strcmp(argv[1], "show") == 0) - backup_subcmd = SHOW_CMD; - else if (strcmp(argv[1], "set-config") == 0) - backup_subcmd = SET_CONFIG_CMD; - else if (strcmp(argv[1], "set-backup") == 0) - backup_subcmd = SET_BACKUP_CMD; - else if (strcmp(argv[1], "show-config") == 0) - backup_subcmd = SHOW_CONFIG_CMD; - else if (strcmp(argv[1], "checkdb") == 0) - backup_subcmd = CHECKDB_CMD; -#ifdef WIN32 - else if (strcmp(argv[1], "ssh") == 0) - launch_ssh(argv); -#endif - else if (strcmp(argv[1], "agent") == 0) - { - /* 'No forward compatibility' sanity: - * /old/binary -> ssh execute -> /newer/binary agent version_num - * If we are executed as an agent for older binary, then exit with error - */ - if (argc > 2) - { - elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " - "is launched as an agent for pg_probackup binary with version '%s'", - PROGRAM_VERSION, argv[2]); - } - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - } - else if (strcmp(argv[1], "--help") == 0 || - strcmp(argv[1], "-?") == 0 || - strcmp(argv[1], "help") == 0) - { - if (argc > 2) - help_command(argv[2]); - else - help_pg_probackup(); - } - else if (strcmp(argv[1], "--version") == 0 - || strcmp(argv[1], "version") == 0 - || strcmp(argv[1], "-V") == 0) + backup_subcmd = parse_subcmd(argv[1]); + switch(backup_subcmd) { -#ifdef PGPRO_VERSION - fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); + case SSH_CMD: +#ifdef WIN32 + launch_ssh(argv); + break; #else - fprintf(stdout, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); + elog(ERROR, "\"ssh\" command implemented only for Windows"); + break; #endif - exit(0); + case AGENT_CMD: + /* 'No forward compatibility' sanity: + * /old/binary -> ssh execute -> /newer/binary agent version_num + * If we are executed as an agent for older binary, then exit with error + */ + if (argc > 2) + elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " + "is launched as an agent for pg_probackup binary with version '%s'", + PROGRAM_VERSION, argv[2]); + remote_agent = true; + fio_communicate(STDIN_FILENO, STDOUT_FILENO); + return 0; + case HELP_CMD: + if (argc > 2) + { + /* 'pg_probackup help command' style */ + help_command(parse_subcmd(argv[2])); + exit(0); + } + else + { + help_pg_probackup(); + exit(0); + } + break; + case VERSION_CMD: + help_print_version(); + exit(0); + case NO_CMD: + elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); + default: + /* Silence compiler warnings */ + break; } - else - elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); } - - if (backup_subcmd == NO_CMD) - elog(ERROR, "No subcommand specified"); + else + elog(ERROR, "No subcommand specified. Please run with \"help\" argument to see possible subcommands."); /* * Make command string before getopt_long() will call. It permutes the * content of argv. */ /* TODO why do we do that only for some commands? */ - command_name = pstrdup(argv[1]); if (backup_subcmd == BACKUP_CMD || backup_subcmd == RESTORE_CMD || backup_subcmd == VALIDATE_CMD || @@ -438,12 +437,34 @@ main(int argc, char *argv[]) /* Parse command line only arguments */ config_get_opt(argc, argv, cmd_options, instance_options); + if (backup_subcmd == SET_CONFIG_CMD) + { + int i; + for (i = 0; i < argc; i++) + { + if (strncmp("--log-format-console", argv[i], strlen("--log-format-console")) == 0) + { + elog(ERROR, "Option 'log-format-console' set only from terminal\n"); + } + } + } + pgut_init(); + if (no_color) + show_color = false; + if (help_opt) - help_command(command_name); + { + /* 'pg_probackup command --help' style */ + help_command(backup_subcmd); + exit(0); + } + + /* set location based on cmdline options only */ + setMyLocation(backup_subcmd); - /* backup_path is required for all pg_probackup commands except help and checkdb */ + /* ===== catalogState ======*/ if (backup_path == NULL) { /* @@ -451,12 +472,8 @@ main(int argc, char *argv[]) * from environment variable */ backup_path = getenv("BACKUP_PATH"); - if (backup_path == NULL && backup_subcmd != CHECKDB_CMD) - elog(ERROR, "required parameter not specified: BACKUP_PATH (-B, --backup-path)"); } - setMyLocation(); - if (backup_path != NULL) { canonicalize_path(backup_path); @@ -464,26 +481,55 @@ main(int argc, char *argv[]) /* Ensure that backup_path is an absolute path */ if (!is_absolute_path(backup_path)) elog(ERROR, "-B, --backup-path must be an absolute path"); + + catalogState = pgut_new(CatalogState); + strncpy(catalogState->catalog_path, backup_path, MAXPGPATH); + join_path_components(catalogState->backup_subdir_path, + catalogState->catalog_path, BACKUPS_DIR); + join_path_components(catalogState->wal_subdir_path, + catalogState->catalog_path, WAL_SUBDIR); } - /* Ensure that backup_path is an absolute path */ - if (backup_path && !is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); + /* backup_path is required for all pg_probackup commands except help, version, checkdb and catchup */ + if (backup_path == NULL && + backup_subcmd != CHECKDB_CMD && + backup_subcmd != HELP_CMD && + backup_subcmd != VERSION_CMD && + backup_subcmd != CATCHUP_CMD) + elog(ERROR, + "No backup catalog path specified.\n" + "Please specify it either using environment variable BACKUP_PATH or\n" + "command line option --backup-path (-B)"); + /* ===== catalogState (END) ======*/ + + /* ===== instanceState ======*/ /* * Option --instance is required for all commands except - * init, show, checkdb and validate + * init, show, checkdb, validate and catchup */ if (instance_name == NULL) { if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD) - elog(ERROR, "required parameter not specified: --instance"); + backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD && backup_subcmd != CATCHUP_CMD) + elog(ERROR, "Required parameter not specified: --instance"); } else - /* Set instance name */ - instance_config.name = pgut_strdup(instance_name); + { + instanceState = pgut_new(InstanceState); + instanceState->catalog_state = catalogState; + + strncpy(instanceState->instance_name, instance_name, MAXPGPATH); + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + + } + /* ===== instanceState (END) ======*/ /* * If --instance option was passed, construct paths for backup data and @@ -491,28 +537,6 @@ main(int argc, char *argv[]) */ if ((backup_path != NULL) && instance_name) { - /* - * Fill global variables used to generate pathes inside the instance's - * backup catalog. - * TODO replace global variables with InstanceConfig structure fields - */ - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); - - /* - * Fill InstanceConfig structure fields used to generate pathes inside - * the instance's backup catalog. - * TODO continue refactoring to use these fields instead of global vars - */ - sprintf(instance_config.backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - canonicalize_path(instance_config.backup_instance_path); - - sprintf(instance_config.arclog_path, "%s/%s/%s", - backup_path, "wal", instance_name); - canonicalize_path(instance_config.arclog_path); - /* * Ensure that requested backup instance exists. * for all commands except init, which doesn't take this parameter, @@ -524,10 +548,11 @@ main(int argc, char *argv[]) { struct stat st; - if (fio_stat(backup_instance_path, &st, true, FIO_BACKUP_HOST) != 0) + if (fio_stat(instanceState->instance_backup_subdir_path, + &st, true, FIO_BACKUP_HOST) != 0) { elog(WARNING, "Failed to access directory \"%s\": %s", - backup_instance_path, strerror(errno)); + instanceState->instance_backup_subdir_path, strerror(errno)); // TODO: redundant message, should we get rid of it? elog(ERROR, "Instance '%s' does not exist in this backup catalog", @@ -549,7 +574,6 @@ main(int argc, char *argv[]) */ if (instance_name) { - char path[MAXPGPATH]; /* Read environment variables */ config_get_opt_env(instance_options); @@ -557,15 +581,22 @@ main(int argc, char *argv[]) if (backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != ARCHIVE_GET_CMD) { - join_path_components(path, backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - if (backup_subcmd == CHECKDB_CMD) - config_read_opt(path, instance_options, ERROR, true, true); + config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, true); else - config_read_opt(path, instance_options, ERROR, true, false); + config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, false); + + /* + * We can determine our location only after reading the configuration file, + * unless we are running arcive-push/archive-get - they are allowed to trust + * cmdline only. + */ + setMyLocation(backup_subcmd); } - setMyLocation(); + } + else if (backup_subcmd == CATCHUP_CMD) + { + config_get_opt_env(instance_options); } /* @@ -595,7 +626,17 @@ main(int argc, char *argv[]) backup_path != NULL && instance_name == NULL && instance_config.pgdata == NULL) - elog(ERROR, "required parameter not specified: --instance"); + elog(ERROR, "Required parameter not specified: --instance"); + + /* Check checkdb command options consistency */ + if (backup_subcmd == CHECKDB_CMD && + !need_amcheck) + { + if (heapallindexed) + elog(ERROR, "--heapallindexed can only be used with --amcheck option"); + if (checkunique) + elog(ERROR, "--checkunique can only be used with --amcheck option"); + } /* Usually checkdb for file logging requires log_directory * to be specified explicitly, but if backup_dir and instance name are provided, @@ -609,6 +650,13 @@ main(int argc, char *argv[]) "You must specify --log-directory option when running checkdb with " "--log-level-file option enabled."); + if (backup_subcmd == CATCHUP_CMD && + instance_config.logger.log_level_file != LOG_OFF && + instance_config.logger.log_directory == NULL) + elog(ERROR, "Cannot save catchup logs to a file. " + "You must specify --log-directory option when running catchup with " + "--log-level-file option enabled."); + /* Initialize logger */ init_logger(backup_path, &instance_config.logger); @@ -637,6 +685,7 @@ main(int argc, char *argv[]) if (instance_config.pgdata != NULL) canonicalize_path(instance_config.pgdata); if (instance_config.pgdata != NULL && + (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) && !is_absolute_path(instance_config.pgdata)) elog(ERROR, "-D, --pgdata must be an absolute path"); @@ -667,7 +716,7 @@ main(int argc, char *argv[]) backup_subcmd != SET_BACKUP_CMD && backup_subcmd != SHOW_CMD) elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); current.backup_id = base36dec(backup_id_string); if (current.backup_id == 0) @@ -693,27 +742,26 @@ main(int argc, char *argv[]) */ recovery_target_options = parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_lsn, + target_inclusive, target_tli_string, target_lsn, (target_stop != NULL) ? target_stop : (target_immediate) ? "immediate" : NULL, target_name, target_action); if (force && backup_subcmd != RESTORE_CMD) elog(ERROR, "You cannot specify \"--force\" flag with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); if (force) no_validate = true; - if (replication_slot != NULL) - restore_as_replica = true; - /* keep all params in one structure */ restore_params = pgut_new(pgRestoreParams); restore_params->is_restore = (backup_subcmd == RESTORE_CMD); restore_params->force = force; restore_params->no_validate = no_validate; restore_params->restore_as_replica = restore_as_replica; + restore_params->recovery_settings_mode = DEFAULT; + restore_params->primary_slot_name = replication_slot; restore_params->skip_block_validation = skip_block_validation; restore_params->skip_external_dirs = skip_external_dirs; @@ -721,6 +769,7 @@ main(int argc, char *argv[]) restore_params->partial_restore_type = NONE; restore_params->primary_conninfo = primary_conninfo; restore_params->incremental_mode = incremental_mode; + restore_params->allow_partial_incremental = allow_partial_incremental; /* handle partial restore parameters */ if (datname_exclude_list && datname_include_list) @@ -736,6 +785,21 @@ main(int argc, char *argv[]) restore_params->partial_restore_type = INCLUDE; restore_params->partial_db_list = datname_include_list; } + + if (gl_waldir_path) + { + /* clean up xlog directory name, check it's absolute */ + canonicalize_path(gl_waldir_path); + if (!is_absolute_path(gl_waldir_path)) + { + elog(ERROR, "WAL directory location must be an absolute path"); + } + if (strlen(gl_waldir_path) > MAXPGPATH) + elog(ERROR, "Value specified to --waldir is too long"); + + } + restore_params->waldir = gl_waldir_path; + } /* @@ -768,10 +832,133 @@ main(int argc, char *argv[]) } } + /* checking required options */ + if (backup_subcmd == CATCHUP_CMD) + { + if (catchup_source_pgdata == NULL) + elog(ERROR, "You must specify \"--source-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (catchup_destination_pgdata == NULL) + elog(ERROR, "You must specify \"--destination-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (current.backup_mode == BACKUP_MODE_INVALID) + elog(ERROR, "No backup mode specified.\n" + "Please specify it either using environment variable BACKUP_MODE or\n" + "command line option --backup-mode (-b)"); + if (current.backup_mode != BACKUP_MODE_FULL && current.backup_mode != BACKUP_MODE_DIFF_PTRACK && current.backup_mode != BACKUP_MODE_DIFF_DELTA) + elog(ERROR, "Only \"FULL\", \"PTRACK\" and \"DELTA\" modes are supported with the \"%s\" command", get_subcmd_name(backup_subcmd)); + if (!stream_wal) + elog(INFO, "--stream is required, forcing stream mode"); + current.stream = stream_wal = true; + if (instance_config.external_dir_str) + elog(ERROR, "External directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); + // TODO check instance_config.conn_opt + } + /* sanity */ if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", - command_name); + get_subcmd_name(backup_subcmd)); + + if (backup_subcmd == ARCHIVE_PUSH_CMD) + { + /* Check archive-push parameters and construct archive_push_xlog_dir + * + * There are 4 cases: + * 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files + * (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push()) + * in this case we can use batches and threads + * 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata) + * in this case we can use this path, as well as batches and thread + * 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id + * in this case we can use this path, as well as batches and thread + * (replica for example, see test_archive_push_sanity) + * 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found + * disable optimizations and work with user specified path + */ + bool check_system_id = true; + uint64 system_id; + char current_dir[MAXPGPATH]; + + if (wal_file_name == NULL) + elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); + + if (instance_config.pgdata == NULL) + elog(ERROR, "Cannot read pg_probackup.conf for this instance"); + + /* TODO may be remove in preference of checking inside compress_init()? */ + if (instance_config.compress_alg == PGLZ_COMPRESS) + elog(ERROR, "Cannot use pglz for WAL compression"); + + if (!getcwd(current_dir, sizeof(current_dir))) + elog(ERROR, "getcwd() error"); + + if (wal_file_path == NULL) + { + /* 1st case */ + system_id = get_system_identifier(current_dir, FIO_DB_HOST, false); + join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR); + } + else + { + /* + * Usually we get something like + * wal_file_path = "pg_wal/0000000100000000000000A1" + * wal_file_name = "0000000100000000000000A1" + * instance_config.pgdata = "/pgdata/.../node/data" + * We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata + * and compare this directories. + * Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving) + */ + char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); + join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); + if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST)) + { + /* 2nd case */ + system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false); + /* archive_push_xlog_dir already have right value */ + } + else + { + if (strlen(stripped_wal_file_path) < MAXPGPATH) + strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH); + else + elog(ERROR, "Value specified to --wal_file_path is too long"); + + system_id = get_system_identifier(current_dir, FIO_DB_HOST, true); + /* 3rd case if control file present -- i.e. system_id != 0 */ + + if (system_id == 0) + { + /* 4th case */ + check_system_id = false; + + if (batch_size > 1 || num_threads > 1 || !no_ready_rename) + { + elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename"); + batch_size = 1; + num_threads = 1; + no_ready_rename = true; + } + } + } + pfree(stripped_wal_file_path); + } + + if (check_system_id && system_id != instance_config.system_identifier) + elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." + "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, + wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id); + } + +#if PG_VERSION_NUM >= 100000 + if (temp_slot && perm_slot) + elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option"); + + /* if slot name was not provided for temp slot, use default slot name */ + if (!replication_slot && temp_slot) + replication_slot = DEFAULT_TEMP_SLOT_NAME; +#endif + if (!replication_slot && perm_slot) + replication_slot = DEFAULT_PERMANENT_SLOT_NAME; if (num_threads < 1) num_threads = 1; @@ -779,40 +966,48 @@ main(int argc, char *argv[]) if (batch_size < 1) batch_size = 1; - compress_init(); + compress_init(backup_subcmd); /* do actual operation */ switch (backup_subcmd) { case ARCHIVE_PUSH_CMD: - do_archive_push(&instance_config, wal_file_path, wal_file_name, + do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name, batch_size, file_overwrite, no_sync, no_ready_rename); break; case ARCHIVE_GET_CMD: - do_archive_get(&instance_config, prefetch_dir, + do_archive_get(instanceState, &instance_config, prefetch_dir, wal_file_path, wal_file_name, batch_size, !no_validate_wal); break; case ADD_INSTANCE_CMD: - return do_add_instance(&instance_config); + return do_add_instance(instanceState, &instance_config); case DELETE_INSTANCE_CMD: - return do_delete_instance(); + return do_delete_instance(instanceState); case INIT_CMD: - return do_init(); + return do_init(catalogState); case BACKUP_CMD: { - time_t start_time = time(NULL); - current.stream = stream_wal; + if (start_time != INVALID_BACKUP_ID) + elog(WARNING, "Please do not use the --start-time option to start backup. " + "This is a service option required to work with other extensions. " + "We do not guarantee future support for this flag."); + /* sanity */ if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "required parameter not specified: BACKUP_MODE " - "(-b, --backup-mode)"); + elog(ERROR, "No backup mode specified.\n" + "Please specify it either using environment variable BACKUP_MODE or\n" + "command line option --backup-mode (-b)"); - return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs); + return do_backup(instanceState, set_backup_params, + no_validate, no_sync, backup_logs, start_time); } + case CATCHUP_CMD: + return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync, + exclude_absolute_paths_list, exclude_relative_paths_list); case RESTORE_CMD: - return do_restore_or_validate(current.backup_id, + return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, restore_params, no_sync); case VALIDATE_CMD: @@ -822,17 +1017,18 @@ main(int argc, char *argv[]) if (datname_exclude_list || datname_include_list) elog(ERROR, "You must specify parameter (-i, --backup-id) for partial validation"); - return do_validate_all(); + return do_validate_all(catalogState, instanceState); } else /* PITR validation and, optionally, partial validation */ - return do_restore_or_validate(current.backup_id, + return do_restore_or_validate(instanceState, current.backup_id, recovery_target_options, restore_params, no_sync); case SHOW_CMD: - return do_show(instance_name, current.backup_id, show_archive); + return do_show(catalogState, instanceState, current.backup_id, show_archive); case DELETE_CMD: + if (delete_expired && backup_id_string) elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); if (merge_expired && backup_id_string) @@ -845,26 +1041,26 @@ main(int argc, char *argv[]) if (!backup_id_string) { if (delete_status) - do_delete_status(&instance_config, delete_status); + do_delete_status(instanceState, &instance_config, delete_status); else - do_retention(); + do_retention(instanceState, no_validate, no_sync); } else - do_delete(current.backup_id); + do_delete(instanceState, current.backup_id); break; case MERGE_CMD: - do_merge(current.backup_id); + do_merge(instanceState, current.backup_id, no_validate, no_sync); break; case SHOW_CONFIG_CMD: - do_show_config(); + do_show_config(show_base_units); break; case SET_CONFIG_CMD: - do_set_config(false); + do_set_config(instanceState, false); break; case SET_BACKUP_CMD: if (!backup_id_string) elog(ERROR, "You must specify parameter (-i, --backup-id) for 'set-backup' command"); - do_set_backup(instance_name, current.backup_id, set_backup_params); + do_set_backup(instanceState, current.backup_id, set_backup_params); break; case CHECKDB_CMD: do_checkdb(need_amcheck, @@ -873,8 +1069,17 @@ main(int argc, char *argv[]) case NO_CMD: /* Should not happen */ elog(ERROR, "Unknown subcommand"); + case SSH_CMD: + case AGENT_CMD: + /* Может перейти на использование какого-нибудь do_agent() для однобразия? */ + case HELP_CMD: + case VERSION_CMD: + /* Silence compiler warnings, these already handled earlier */ + break; } + free_environment_locale(); + return 0; } @@ -935,13 +1140,13 @@ opt_show_format(ConfigOption *opt, const char *arg) * Initialize compress and sanity checks for compress. */ static void -compress_init(void) +compress_init(ProbackupSubcmd const subcmd) { /* Default algorithm is zlib */ if (compress_shortcut) instance_config.compress_alg = ZLIB_COMPRESS; - if (backup_subcmd != SET_CONFIG_CMD) + if (subcmd != SET_CONFIG_CMD) { if (instance_config.compress_level != COMPRESS_LEVEL_DEFAULT && instance_config.compress_alg == NOT_DEFINED_COMPRESS) @@ -955,7 +1160,7 @@ compress_init(void) if (instance_config.compress_alg == ZLIB_COMPRESS && instance_config.compress_level == 0) elog(WARNING, "Compression level 0 will lead to data bloat!"); - if (backup_subcmd == BACKUP_CMD || backup_subcmd == ARCHIVE_PUSH_CMD) + if (subcmd == BACKUP_CMD || subcmd == ARCHIVE_PUSH_CMD) { #ifndef HAVE_LIBZ if (instance_config.compress_alg == ZLIB_COMPRESS) @@ -967,39 +1172,45 @@ compress_init(void) } } -/* Construct array of datnames, provided by user via db-exclude option */ -void -opt_datname_exclude_list(ConfigOption *opt, const char *arg) +static void +opt_parser_add_to_parray_helper(parray **list, const char *str) { - char *dbname = NULL; + char *elem = NULL; - if (!datname_exclude_list) - datname_exclude_list = parray_new(); + if (*list == NULL) + *list = parray_new(); - dbname = pgut_malloc(strlen(arg) + 1); + elem = pgut_malloc(strlen(str) + 1); + strcpy(elem, str); - /* TODO add sanity for database name */ - strcpy(dbname, arg); + parray_append(*list, elem); +} - parray_append(datname_exclude_list, dbname); +/* Construct array of datnames, provided by user via db-exclude option */ +void +opt_datname_exclude_list(ConfigOption *opt, const char *arg) +{ + /* TODO add sanity for database name */ + opt_parser_add_to_parray_helper(&datname_exclude_list, arg); } /* Construct array of datnames, provided by user via db-include option */ void opt_datname_include_list(ConfigOption *opt, const char *arg) { - char *dbname = NULL; - - if (!datname_include_list) - datname_include_list = parray_new(); - - dbname = pgut_malloc(strlen(arg) + 1); - - if (strcmp(dbname, "tempate0") == 0 || - strcmp(dbname, "tempate1") == 0) + if (strcmp(arg, "template0") == 0 || + strcmp(arg, "template1") == 0) elog(ERROR, "Databases 'template0' and 'template1' cannot be used for partial restore or validation"); - strcpy(dbname, arg); + opt_parser_add_to_parray_helper(&datname_include_list, arg); +} - parray_append(datname_include_list, dbname); +/* Parse --exclude-path option */ +void +opt_exclude_path(ConfigOption *opt, const char *arg) +{ + if (is_absolute_path(arg)) + opt_parser_add_to_parray_helper(&exclude_absolute_paths_list, arg); + else + opt_parser_add_to_parray_helper(&exclude_relative_paths_list, arg); } diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b5bdd5c21..ae99e0605 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,19 +3,21 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2018, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ #ifndef PG_PROBACKUP_H #define PG_PROBACKUP_H + #include "postgres_fe.h" #include "libpq-fe.h" #include "libpq-int.h" #include "access/xlog_internal.h" #include "utils/pg_crc.h" +#include "catalog/pg_control.h" #if PG_VERSION_NUM >= 120000 #include "common/logging.h" @@ -39,12 +41,24 @@ #include "datapagemap.h" #include "utils/thread.h" +#include "pg_probackup_state.h" + + #ifdef WIN32 #define __thread __declspec(thread) #else #include #endif +#if PG_VERSION_NUM >= 150000 +// _() is explicitly undefined in libpq-int.h +// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0 +#define _(s) gettext(s) +#endif + +/* Wrap the code that we're going to delete after refactoring in this define*/ +#define REFACTORE_ME + /* pgut client variables and full path */ extern const char *PROGRAM_NAME; extern const char *PROGRAM_NAME_FULL; @@ -55,6 +69,7 @@ extern const char *PROGRAM_EMAIL; /* Directory/File names */ #define DATABASE_DIR "database" #define BACKUPS_DIR "backups" +#define WAL_SUBDIR "wal" #if PG_VERSION_NUM >= 100000 #define PG_XLOG_DIR "pg_wal" #define PG_LOG_DIR "log" @@ -66,18 +81,28 @@ extern const char *PROGRAM_EMAIL; #define PG_GLOBAL_DIR "global" #define BACKUP_CONTROL_FILE "backup.control" #define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" -#define BACKUP_CATALOG_PID "backup.pid" +#define BACKUP_LOCK_FILE "backup.pid" +#define BACKUP_RO_LOCK_FILE "backup_ro.pid" #define DATABASE_FILE_LIST "backup_content.control" #define PG_BACKUP_LABEL_FILE "backup_label" -#define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define PG_TABLESPACE_MAP_FILE "tablespace_map" +#define RELMAPPER_FILENAME "pg_filenode.map" #define EXTERNAL_DIR "external_directories/externaldir" #define DATABASE_MAP "database_map" #define HEADER_MAP "page_header_map" #define HEADER_MAP_TMP "page_header_map_tmp" +#define XLOG_CONTROL_BAK_FILE XLOG_CONTROL_FILE".pbk.bak" + +/* default replication slot names */ +#define DEFAULT_TEMP_SLOT_NAME "pg_probackup_slot"; +#define DEFAULT_PERMANENT_SLOT_NAME "pg_probackup_perm_slot"; /* Timeout defaults */ #define ARCHIVE_TIMEOUT_DEFAULT 300 #define REPLICA_TIMEOUT_DEFAULT 300 +#define LOCK_TIMEOUT 60 +#define LOCK_STALE_TIMEOUT 30 +#define LOG_FREQ 10 /* Directory/File permission */ #define DIR_PERMISSION (0700) @@ -86,6 +111,8 @@ extern const char *PROGRAM_EMAIL; /* 64-bit xid support for PGPRO_EE */ #ifndef PGPRO_EE #define XID_FMT "%u" +#elif !defined(XID_FMT) +#define XID_FMT UINT64_FORMAT #endif #ifndef STDIN_FILENO @@ -111,6 +138,25 @@ extern const char *PROGRAM_EMAIL; #define XRecOffIsNull(xlrp) \ ((xlrp) % XLOG_BLCKSZ == 0) +/* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ +#define base36bufsize 14 + +/* Text Coloring macro */ +#define TC_LEN 11 +#define TC_RED "\033[0;31m" +#define TC_RED_BOLD "\033[1;31m" +#define TC_BLUE "\033[0;34m" +#define TC_BLUE_BOLD "\033[1;34m" +#define TC_GREEN "\033[0;32m" +#define TC_GREEN_BOLD "\033[1;32m" +#define TC_YELLOW "\033[0;33m" +#define TC_YELLOW_BOLD "\033[1;33m" +#define TC_MAGENTA "\033[0;35m" +#define TC_MAGENTA_BOLD "\033[1;35m" +#define TC_CYAN "\033[0;36m" +#define TC_CYAN_BOLD "\033[1;36m" +#define TC_RESET "\033[0m" + typedef struct RedoParams { TimeLineID tli; @@ -130,6 +176,17 @@ typedef struct db_map_entry char *datname; } db_map_entry; +/* State of pgdata in the context of its compatibility for incremental restore */ +typedef enum DestDirIncrCompatibility +{ + POSTMASTER_IS_RUNNING, + SYSTEM_ID_MISMATCH, + BACKUP_LABEL_EXISTS, + PARTIAL_INCREMENTAL_FORBIDDEN, + DEST_IS_NOT_OK, + DEST_OK +} DestDirIncrCompatibility; + typedef enum IncrRestoreMode { INCR_NONE, @@ -144,6 +201,16 @@ typedef enum PartialRestoreType EXCLUDE, } PartialRestoreType; +typedef enum RecoverySettingsMode +{ + DEFAULT, /* not set */ + DONTWRITE, /* explicitly forbid to update recovery settings */ + //TODO Should we always clean/preserve old recovery settings, + // or make it configurable? + PITR_REQUESTED, /* can be set based on other parameters + * if not explicitly forbidden */ +} RecoverySettingsMode; + typedef enum CompressAlg { NOT_DEFINED_COMPRESS = 0, @@ -154,11 +221,14 @@ typedef enum CompressAlg typedef enum ForkName { + none, vm, fsm, cfm, init, - ptrack + ptrack, + cfs_bck, + cfm_bck } ForkName; #define INIT_FILE_CRC32(use_crc32c, crc) \ @@ -183,6 +253,8 @@ do { \ FIN_TRADITIONAL_CRC32(crc); \ } while (0) +#define pg_off_t unsigned long long + /* Information about single file (or dir) in backup */ typedef struct pgFile @@ -212,7 +284,7 @@ typedef struct pgFile int segno; /* Segment number for ptrack */ int n_blocks; /* number of blocks in the data file in data directory */ bool is_cfs; /* Flag to distinguish files compressed by CFS*/ - bool is_database; /* Flag used strictly by ptrack 1.x backup */ + struct pgFile *cfs_chain; /* linked list of CFS segment's cfm, bck, cfm_bck related files */ int external_dir_num; /* Number of external directory. 0 if not external */ bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ CompressAlg compress_alg; /* compression algorithm applied to the file */ @@ -224,8 +296,11 @@ typedef struct pgFile /* Coordinates in header map */ int n_headers; /* number of blocks in the data file in backup */ pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ - off_t hdr_off; /* offset in header map */ - int hdr_size; /* offset in header map */ + pg_off_t hdr_off; /* offset in header map */ + int hdr_size; /* length of headers */ + bool excluded; /* excluded via --exclude-path option */ + bool skip_cfs_nested; /* mark to skip in processing treads as nested to cfs_chain */ + bool remove_from_list; /* tmp flag to clean up files list from temp and unlogged tables */ } pgFile; typedef struct page_map_entry @@ -238,6 +313,11 @@ typedef struct page_map_entry /* Special values of datapagemap_t bitmapsize */ #define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ +/* Return codes for check_tablespace_mapping */ +#define NoTblspc 0 +#define EmptyTblspc 1 +#define NotEmptyTblspc 2 + /* Current state of backup */ typedef enum BackupStatus { @@ -276,9 +356,14 @@ typedef enum ShowFormat #define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ #define FILE_NOT_FOUND (-2) /* file disappeared during backup */ #define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.4.1" -#define AGENT_PROTOCOL_VERSION 20401 +#define PROGRAM_VERSION "2.5.15" + +/* update when remote agent API or behaviour changes */ +#define AGENT_PROTOCOL_VERSION 20509 +#define AGENT_PROTOCOL_VERSION_STR "2.5.9" +/* update only when changing storage format */ +#define STORAGE_FORMAT_VERSION "2.4.4" typedef struct ConnectionOptions { @@ -308,10 +393,6 @@ typedef struct ArchiveOptions */ typedef struct InstanceConfig { - char *name; - char arclog_path[MAXPGPATH]; - char backup_instance_path[MAXPGPATH]; - uint64 system_identifier; uint32 xlog_seg_size; @@ -321,7 +402,7 @@ typedef struct InstanceConfig ConnectionOptions conn_opt; ConnectionOptions master_conn_opt; - uint32 replica_timeout; + uint32 replica_timeout; //Deprecated. Not used anywhere /* Wait timeout for WAL segment archiving */ uint32 archive_timeout; @@ -363,7 +444,7 @@ typedef struct PGNodeInfo char server_version_str[100]; int ptrack_version_num; - bool is_ptrack_enable; + bool is_ptrack_enabled; const char *ptrack_schema; /* used only for ptrack 2.x */ } PGNodeInfo; @@ -371,11 +452,11 @@ typedef struct PGNodeInfo /* structure used for access to block header map */ typedef struct HeaderMap { - char path[MAXPGPATH]; - char path_tmp[MAXPGPATH]; /* used only in merge */ - FILE *fp; /* used only for writing */ - char *buf; /* buffer */ - off_t offset; /* current position in fp */ + char path[MAXPGPATH]; + char path_tmp[MAXPGPATH]; /* used only in merge */ + FILE *fp; /* used only for writing */ + char *buf; /* buffer */ + pg_off_t offset; /* current position in fp */ pthread_mutex_t mutex; } HeaderMap; @@ -387,15 +468,17 @@ struct pgBackup { BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ time_t backup_id; /* Identifier of the backup. - * Currently it's the same as start_time */ + * By default it's the same as start_time + * but can be increased if same backup_id + * already exists. It can be also set by + * start_time parameter */ BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ TimeLineID tli; /* timeline of start and stop backup lsns */ XLogRecPtr start_lsn; /* backup's starting transaction log location */ XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ - time_t start_time; /* since this moment backup has status - * BACKUP_STATUS_RUNNING */ - time_t merge_dest_backup; /* start_time of incremental backup, - * this backup is merging with. + time_t start_time; /* UTC time of backup creation */ + time_t merge_dest_backup; /* start_time of incremental backup with + * which this backup is merging with. * Only available for FULL backups * with MERGING or MERGED statuses */ time_t merge_time; /* the moment when merge was started or 0 */ @@ -460,6 +543,8 @@ struct pgBackup /* map used for access to page headers */ HeaderMap hdr_map; + + char backup_id_encoded[base36bufsize]; }; /* Recovery target for restore and validate subcommands */ @@ -480,6 +565,7 @@ typedef struct pgRecoveryTarget const char *target_stop; const char *target_name; const char *target_action; + const char *target_tli_string; /* timeline number, "current" or "latest" from recovery_target_timeline option*/ } pgRecoveryTarget; /* Options needed for restore and validate commands */ @@ -489,6 +575,8 @@ typedef struct pgRestoreParams bool is_restore; bool no_validate; bool restore_as_replica; + //TODO maybe somehow add restore_as_replica as one of RecoverySettingsModes + RecoverySettingsMode recovery_settings_mode; bool skip_external_dirs; bool skip_block_validation; //Start using it const char *restore_command; @@ -502,6 +590,9 @@ typedef struct pgRestoreParams /* options for partial restore */ PartialRestoreType partial_restore_type; parray *partial_db_list; + bool allow_partial_incremental; + + char* waldir; } pgRestoreParams; /* Options needed for set-backup command */ @@ -530,7 +621,6 @@ typedef struct parray *external_dirs; XLogRecPtr prev_start_lsn; - ConnectionArgs conn_arg; int thread_num; HeaderMap *hdr_map; @@ -541,7 +631,6 @@ typedef struct int ret; } backup_files_arg; - typedef struct timelineInfo timelineInfo; /* struct to collect info about timelines in WAL archive */ @@ -620,22 +709,18 @@ typedef struct BackupPageHeader2 uint16 checksum; } BackupPageHeader2; +typedef struct StopBackupCallbackParams +{ + PGconn *conn; + int server_version; +} StopBackupCallbackParams; + /* Special value for compressed_size field */ #define PageIsOk 0 #define SkipCurrentPage -1 #define PageIsTruncated -2 #define PageIsCorrupted -3 /* used by checkdb */ - -/* - * return pointer that exceeds the length of prefix from character string. - * ex. str="/xxx/yyy/zzz", prefix="/xxx/yyy", return="zzz". - * - * Deprecated. Do not use this in new code. - */ -#define GetRelativePath(str, prefix) \ - ((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1) - /* * Return timeline, xlog ID and record offset from an LSN of the type * 0/B000188, usual result from pg_stop_backup() and friends. @@ -649,6 +734,9 @@ typedef struct BackupPageHeader2 strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) #if PG_VERSION_NUM >= 110000 + +#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ + XLogSegmentOffset(xlogptr, wal_segsz_bytes) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) #define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ @@ -668,6 +756,8 @@ typedef struct BackupPageHeader2 #define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) #else +#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ + ((xlogptr) & ((XLogSegSize) - 1)) #define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ XLByteToSeg(xlrp, logSegNo) #define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ @@ -698,6 +788,11 @@ typedef struct BackupPageHeader2 strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0) +#define IsTempPartialXLogFileName(fname) \ + (strlen(fname) == XLOG_FNAME_LEN + strlen(".partial.part") && \ + strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ + strcmp((fname) + XLOG_FNAME_LEN, ".partial.part") == 0) + #define IsTempCompressXLogFileName(fname) \ (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \ strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ @@ -705,29 +800,26 @@ typedef struct BackupPageHeader2 #define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) -/* directory options */ -extern char *backup_path; -extern char backup_instance_path[MAXPGPATH]; -extern char arclog_path[MAXPGPATH]; - /* common options */ extern pid_t my_pid; extern __thread int my_thread_num; extern int num_threads; extern bool stream_wal; +extern bool show_color; extern bool progress; extern bool is_archive_cmd; /* true for archive-{get,push} */ -#if PG_VERSION_NUM >= 100000 /* In pre-10 'replication_slot' is defined in receivelog.h */ extern char *replication_slot; -#endif +#if PG_VERSION_NUM >= 100000 extern bool temp_slot; +#endif +extern bool perm_slot; /* backup options */ extern bool smooth_checkpoint; /* remote probackup options */ -extern char* remote_agent; +extern bool remote_agent; extern bool exclusive_backup; @@ -737,17 +829,39 @@ extern bool delete_expired; extern bool merge_expired; extern bool dry_run; -/* compression options */ -extern bool compress_shortcut; +/* ===== instanceState ===== */ + +typedef struct InstanceState +{ + /* catalog, this instance belongs to */ + CatalogState *catalog_state; + + char instance_name[MAXPGPATH]; //previously global var instance_name + /* $BACKUP_PATH/backups/instance_name */ + char instance_backup_subdir_path[MAXPGPATH]; + + /* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ + char instance_config_path[MAXPGPATH]; + + /* $BACKUP_PATH/backups/instance_name */ + char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path -/* other options */ -extern char *instance_name; + /* TODO: Make it more specific */ + PGconn *conn; + + + //TODO split into some more meaningdul parts + InstanceConfig *config; +} InstanceState; + +/* ===== instanceState (END) ===== */ /* show options */ extern ShowFormat show_format; /* checkdb options */ extern bool heapallindexed; +extern bool checkunique; extern bool skip_block_validation; /* current settings */ @@ -756,13 +870,9 @@ extern pgBackup current; /* argv of the process */ extern char** commands_args; -/* in dir.c */ -/* exclude directory list for $PGDATA file listing */ -extern const char *pgdata_exclude_dir[]; - /* in backup.c */ -extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs); +extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, + bool no_validate, bool no_sync, bool backup_logs, time_t start_time); extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, char *pgdata); extern BackupMode parse_backup_mode(const char *value); @@ -770,65 +880,83 @@ extern const char *deparse_backup_mode(BackupMode mode); extern void process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno); -extern char *pg_ptrack_get_block(ConnectionArgs *arguments, - Oid dbOid, Oid tblsOid, Oid relOid, - BlockNumber blknum, size_t *result_size, - int ptrack_version_num, const char *ptrack_schema); +/* in catchup.c */ +extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, + parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list); + /* in restore.c */ -extern int do_restore_or_validate(time_t target_backup_id, +extern int do_restore_or_validate(InstanceState *instanceState, + time_t target_backup_id, pgRecoveryTarget *rt, pgRestoreParams *params, bool no_sync); -extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup); +extern bool satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn); extern bool satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt); extern pgRecoveryTarget *parseRecoveryTargetOptions( const char *target_time, const char *target_xid, - const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, + const char *target_inclusive, const char *target_tli_string, const char* target_lsn, const char *target_stop, const char *target_name, const char *target_action); extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, PartialRestoreType partial_restore_type); +extern const char* backup_id_of(pgBackup *backup); +extern void reset_backup_id(pgBackup *backup); + extern parray *get_backup_filelist(pgBackup *backup, bool strict); extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); +extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, + IncrRestoreMode incremental_mode, + parray *partial_db_list, + bool allow_partial_incremental); + +/* in remote.c */ +extern void check_remote_agent_compatibility(int agent_version, + char *compatibility_str, size_t compatibility_str_max_size); +extern size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size); /* in merge.c */ -extern void do_merge(time_t backup_id); +extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); extern void merge_backups(pgBackup *backup, pgBackup *next_backup); -extern void merge_chain(parray *parent_chain, - pgBackup *full_backup, pgBackup *dest_backup); +extern void merge_chain(InstanceState *instanceState, parray *parent_chain, + pgBackup *full_backup, pgBackup *dest_backup, + bool no_validate, bool no_sync); extern parray *read_database_map(pgBackup *backup); /* in init.c */ -extern int do_init(void); -extern int do_add_instance(InstanceConfig *instance); +extern int do_init(CatalogState *catalogState); +extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance); /* in archive.c */ -extern void do_archive_push(InstanceConfig *instance, char *wal_file_path, +extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, char *wal_file_name, int batch_size, bool overwrite, bool no_sync, bool no_ready_rename); -extern void do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, +extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, char *wal_file_name, int batch_size, bool validate_wal); /* in configure.c */ -extern void do_show_config(void); -extern void do_set_config(bool missing_ok); +extern void do_show_config(bool show_base_units); +extern void do_set_config(InstanceState *instanceState, bool missing_ok); extern void init_config(InstanceConfig *config, const char *instance_name); -extern InstanceConfig *readInstanceConfigFile(const char *instance_name); +extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); /* in show.c */ -extern int do_show(const char *instance_name, time_t requested_backup_id, bool show_archive); +extern int do_show(CatalogState *catalogState, InstanceState *instanceState, + time_t requested_backup_id, bool show_archive); +extern void memorize_environment_locale(void); +extern void free_environment_locale(void); /* in delete.c */ -extern void do_delete(time_t backup_id); +extern void do_delete(InstanceState *instanceState, time_t backup_id); extern void delete_backup_files(pgBackup *backup); -extern void do_retention(void); -extern int do_delete_instance(void); -extern void do_delete_status(InstanceConfig *instance_config, const char *status); +extern void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync); +extern int do_delete_instance(InstanceState *instanceState); +extern void do_delete_status(InstanceState *instanceState, + InstanceConfig *instance_config, const char *status); /* in fetch.c */ extern char *slurpFile(const char *datadir, @@ -839,15 +967,19 @@ extern char *slurpFile(const char *datadir, extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ +extern void help_print_version(void); extern void help_pg_probackup(void); -extern void help_command(char *command); +extern void help_command(ProbackupSubcmd const subcmd); /* in validate.c */ extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); -extern int do_validate_all(void); +extern int do_validate_all(CatalogState *catalogState, InstanceState *instanceState); extern int validate_one_page(Page page, BlockNumber absolute_blkno, XLogRecPtr stop_lsn, PageState *page_st, uint32 checksum_version); +extern bool validate_tablespace_map(pgBackup *backup, bool no_validate); + +extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list); /* return codes for validate_one_page */ /* TODO: use enum */ @@ -862,41 +994,38 @@ extern int validate_one_page(Page page, BlockNumber absolute_blkno, extern pgBackup *read_backup(const char *root_dir); extern void write_backup(pgBackup *backup, bool strict); extern void write_backup_status(pgBackup *backup, BackupStatus status, - const char *instance_name, bool strict); + bool strict); extern void write_backup_data_bytes(pgBackup *backup); -extern bool lock_backup(pgBackup *backup, bool strict); +extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); -extern const char *pgBackupGetBackupMode(pgBackup *backup); +extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color); +extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode); -extern parray *catalog_get_instance_list(void); -extern parray *catalog_get_backup_list(const char *instance_name, time_t requested_backup_id); +extern parray *catalog_get_instance_list(CatalogState *catalogState); + +extern parray *catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id); extern void catalog_lock_backup_list(parray *backup_list, int from_idx, - int to_idx, bool strict); + int to_idx, bool strict, bool exclusive); extern pgBackup *catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time); extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list, TimeLineID current_tli, time_t current_start_time, InstanceConfig *instance); +extern timelineInfo *timelineInfoNew(TimeLineID tli); extern void timelineInfoFree(void *tliInfo); -extern parray *catalog_get_timelines(InstanceConfig *instance); -extern void do_set_backup(const char *instance_name, time_t backup_id, +extern parray *catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance); +extern void do_set_backup(InstanceState *instanceState, time_t backup_id, pgSetBackupParams *set_backup_params); extern void pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params); extern void add_note(pgBackup *target_backup, char *note); -extern void pgBackupWriteControl(FILE *out, pgBackup *backup); +extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); extern void write_backup_filelist(pgBackup *backup, parray *files, const char *root, parray *external_list, bool sync); -extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len, - const char *subdir); -extern void pgBackupGetPath2(const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2); -extern void pgBackupGetPathInInstance(const char *instance_name, - const pgBackup *backup, char *path, size_t len, - const char *subdir1, const char *subdir2); -extern int pgBackupCreateDir(pgBackup *backup); + +extern void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path); extern void pgNodeInit(PGNodeInfo *node); extern void pgBackupInit(pgBackup *backup); extern void pgBackupFree(void *backup); @@ -913,7 +1042,6 @@ extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); extern bool is_prolific(parray *backup_list, pgBackup *target_backup); -extern int get_backup_index_number(parray *backup_list, pgBackup *backup); extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list); extern bool launch_agent(void); extern void launch_ssh(char* argv[]); @@ -926,21 +1054,26 @@ extern CompressAlg parse_compress_alg(const char *arg); extern const char* deparse_compress_alg(int alg); /* in dir.c */ +extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory); +extern bool get_control_value_str(const char *str, const char *name, + char *value_str, size_t value_str_size, bool is_mandatory); extern void dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location); +extern const char *get_tablespace_mapping(const char *dir); extern void create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, bool extract_tablespaces, bool incremental, - fio_location location); + fio_location location, + const char *waldir_path); -extern void read_tablespace_map(parray *files, const char *backup_dir); +extern void read_tablespace_map(parray *links, const char *backup_dir); extern void opt_tablespace_map(ConfigOption *opt, const char *arg); extern void opt_externaldir_map(ConfigOption *opt, const char *arg); -extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty); +extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate); extern void check_external_dir_mapping(pgBackup *backup, bool incremental); extern char *get_external_remap(char *current_dir); @@ -951,8 +1084,6 @@ extern void db_map_entry_free(void *map); extern void print_file_list(FILE *out, const parray *files, const char *root, const char *external_prefix, parray *external_list); -extern parray *dir_read_file_list(const char *root, const char *external_prefix, - const char *file_txt, fio_location location, pg_crc32 expected_crc); extern parray *make_external_directory_list(const char *colon_separated_dirs, bool remap); extern void free_dir_list(parray *list); @@ -960,7 +1091,7 @@ extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_external(const char *dir, parray *dirs_list); -extern int dir_create_dir(const char *path, mode_t mode); +extern int dir_create_dir(const char *path, mode_t mode, bool strict); extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); @@ -975,27 +1106,37 @@ extern void fio_pgFileDelete(pgFile *file, const char *full_path); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool missing_ok, bool use_crc32c); +extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); +extern int pgFileCompareNameWithString(const void *f1, const void *f2); +extern int pgFileCompareRelPathWithString(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); extern int pgFileCompareLinked(const void *f1, const void *f2); extern int pgFileCompareSize(const void *f1, const void *f2); +extern int pgFileCompareSizeDesc(const void *f1, const void *f2); +extern int pgCompareString(const void *str1, const void *str2); +extern int pgPrefixCompareString(const void *str1, const void *str2); extern int pgCompareOid(const void *f1, const void *f2); +extern void pfilearray_clear_locks(parray *file_list); +extern bool set_forkname(pgFile *file); /* in data.c */ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, const char *from_fullpath, uint32 checksum_version); -extern void backup_data_file(ConnectionArgs* conn_arg, pgFile *file, - const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - int ptrack_version_num, const char *ptrack_schema, - HeaderMap *hdr_map, bool missing_ok); + +extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr sync_lsn, BackupMode backup_mode, + uint32 checksum_version, size_t prev_size); +extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, + XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, + CompressAlg calg, int clevel, uint32 checksum_version, + HeaderMap *hdr_map, bool missing_ok); extern void backup_non_data_file(pgFile *file, pgFile *prev_file, const char *from_fullpath, const char *to_fullpath, BackupMode backup_mode, time_t parent_backup_time, @@ -1024,8 +1165,6 @@ extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); -extern pid_t check_postmaster(const char *pgdata); - extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); @@ -1061,24 +1200,28 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(bool safe); +extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path); +extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern uint32 get_xlog_seg_size(char *pgdata_path); -extern void get_redo(const char *pgdata_path, RedoParams *redo); +extern uint32 get_xlog_seg_size(const char *pgdata_path); +extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); +extern void get_control_file_or_back_file(const char *pgdata_path, fio_location location, + ControlFileData *control); extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, const char *to_fullpath, fio_location to_location, pgFile *file); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); +const char *status2str_color(BackupStatus status); extern BackupStatus str2status(const char *status); -extern const char *base36enc(long unsigned int value); -extern char *base36enc_dup(long unsigned int value); +extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]); +/* Abuse C99 Compound Literal's lifetime */ +#define base36enc(value) (base36enc_to((value), (char[base36bufsize]){0})) extern long unsigned int base36dec(const char *text); extern uint32 parse_server_version(const char *server_version_str); extern uint32 parse_program_version(const char *program_version); @@ -1092,24 +1235,17 @@ extern void pretty_size(int64 size, char *buf, size_t len); extern void pretty_time_interval(double time, char *buf, size_t len); extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); -extern void check_system_identifiers(PGconn *conn, char *pgdata); +extern void check_system_identifiers(PGconn *conn, const char *pgdata); extern void parse_filelist_filenames(parray *files, const char *root); /* in ptrack.c */ -extern void make_pagemap_from_ptrack_1(parray* files, PGconn* backup_conn); extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); -extern void pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num); extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num); -extern bool pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn); -extern char *pg_ptrack_get_and_clear(Oid tablespace_oid, - Oid db_oid, - Oid rel_oid, - size_t *result_size, - PGconn *backup_conn); +extern bool pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num); + extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, int ptrack_version_num, XLogRecPtr lsn); @@ -1117,26 +1253,40 @@ extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack /* open local file to writing */ extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size); -extern int send_pages(ConnectionArgs* conn_arg, const char *to_fullpath, const char *from_fullpath, +extern int send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode, int ptrack_version_num, const char *ptrack_schema); + BackupMode backup_mode); +extern int copy_pages(const char *to_fullpath, const char *from_fullpath, + pgFile *file, XLogRecPtr prev_backup_start_lsn, + uint32 checksum_version, bool use_pagemap, + BackupMode backup_mode); /* FIO */ +extern void setMyLocation(ProbackupSubcmd const subcmd); extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); -extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, - int calg, int clevel, uint32 checksum_version, datapagemap_t *pagemap, - BlockNumber* err_blknum, char **errormsg, BackupPageHeader2 **headers); +extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber *err_blknum, char **errormsg, + BackupPageHeader2 **headers); +extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber *err_blknum, char **errormsg); /* return codes for fio_send_pages */ -extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); -extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, +extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg); +extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, pgFile *file, char **errormsg); +extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, + pgFile *file, char **errormsg); extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); +extern void pgut_setenv(const char *key, const char *val); +extern void pgut_unsetenv(const char *key); + extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); @@ -1145,7 +1295,7 @@ extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_vers fio_location location); extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); -extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg); +extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg); /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) @@ -1171,4 +1321,51 @@ datapagemap_is_set(datapagemap_t *map, BlockNumber blkno); extern void datapagemap_print_debug(datapagemap_t *map); +/* in stream.c */ +extern XLogRecPtr stop_backup_lsn; +extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, + ConnectionOptions *conn_opt, + XLogRecPtr startpos, TimeLineID starttli, + bool is_backup); +extern int wait_WAL_streaming_end(parray *backup_files_list); +extern parray* parse_tli_history_buffer(char *history, TimeLineID tli); + +/* external variables and functions, implemented in backup.c */ +typedef struct PGStopBackupResult +{ + /* + * We will use values of snapshot_xid and invocation_time if there are + * no transactions between start_lsn and stop_lsn. + */ + TransactionId snapshot_xid; + time_t invocation_time; + /* + * Fields that store pg_catalog.pg_stop_backup() result + */ + XLogRecPtr lsn; + size_t backup_label_content_len; + char *backup_label_content; + size_t tablespace_map_content_len; + char *tablespace_map_content; +} PGStopBackupResult; + +extern bool backup_in_progress; +extern parray *backup_files_list; + +extern void pg_start_backup(const char *label, bool smooth, pgBackup *backup, + PGNodeInfo *nodeInfo, PGconn *conn); +extern void pg_silent_client_messages(PGconn *conn); +extern void pg_create_restore_point(PGconn *conn, time_t backup_start_time); +extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); +extern void pg_stop_backup_consume(PGconn *conn, int server_version, + bool is_exclusive, uint32 timeout, const char *query_text, + PGStopBackupResult *result); +extern void pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, + const void *data, size_t len, parray *file_list); +extern XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, + bool in_prev_segment, bool segment_only, + int timeout_elevel, bool in_stream_dir); +extern void wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup); +extern int64 calculate_datasize_of_filelist(parray *filelist); + #endif /* PG_PROBACKUP_H */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h new file mode 100644 index 000000000..56d852537 --- /dev/null +++ b/src/pg_probackup_state.h @@ -0,0 +1,31 @@ +/*------------------------------------------------------------------------- + * + * pg_probackup_state.h: Definitions of internal pg_probackup states + * + * Portions Copyright (c) 2021, Postgres Professional + * + *------------------------------------------------------------------------- + */ +#ifndef PG_PROBACKUP_STATE_H +#define PG_PROBACKUP_STATE_H + +/* ====== CatalogState ======= */ + +typedef struct CatalogState +{ + /* $BACKUP_PATH */ + char catalog_path[MAXPGPATH]; //previously global var backup_path + /* $BACKUP_PATH/backups */ + char backup_subdir_path[MAXPGPATH]; + /* $BACKUP_PATH/wal */ + char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path +} CatalogState; + +/* ====== CatalogState (END) ======= */ + + +/* ===== instanceState ===== */ + +/* ===== instanceState (END) ===== */ + +#endif /* PG_PROBACKUP_STATE_H */ diff --git a/src/ptrack.c b/src/ptrack.c index 3f2591137..d27629e45 100644 --- a/src/ptrack.c +++ b/src/ptrack.c @@ -2,7 +2,7 @@ * * ptrack.c: support functions for ptrack backups * - * Copyright (c) 2019 Postgres Professional + * Copyright (c) 2021 Postgres Professional * *------------------------------------------------------------------------- */ @@ -22,121 +22,20 @@ #define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) /* - * Given a list of files in the instance to backup, build a pagemap for each - * data file that has ptrack. Result is saved in the pagemap field of pgFile. - * NOTE we rely on the fact that provided parray is sorted by file->rel_path. + * Parse a string like "2.1" into int + * result: int by formula major_number * 100 + minor_number + * or -1 if string cannot be parsed */ -void -make_pagemap_from_ptrack_1(parray *files, PGconn *backup_conn) +static int +ptrack_parse_version_string(const char *version_str) { - size_t i; - Oid dbOid_with_ptrack_init = 0; - Oid tblspcOid_with_ptrack_init = 0; - char *ptrack_nonparsed = NULL; - size_t ptrack_nonparsed_size = 0; - - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - size_t start_addr; - - /* - * If there is a ptrack_init file in the database, - * we must backup all its files, ignoring ptrack files for relations. - */ - if (file->is_database) - { - /* - * The function pg_ptrack_get_and_clear_db returns true - * if there was a ptrack_init file. - * Also ignore ptrack files for global tablespace, - * to avoid any possible specific errors. - */ - if ((file->tblspcOid == GLOBALTABLESPACE_OID) || - pg_ptrack_get_and_clear_db(file->dbOid, file->tblspcOid, backup_conn)) - { - dbOid_with_ptrack_init = file->dbOid; - tblspcOid_with_ptrack_init = file->tblspcOid; - } - } - - if (file->is_datafile) - { - if (file->tblspcOid == tblspcOid_with_ptrack_init && - file->dbOid == dbOid_with_ptrack_init) - { - /* ignore ptrack if ptrack_init exists */ - elog(VERBOSE, "Ignoring ptrack because of ptrack_init for file: %s", file->rel_path); - file->pagemap_isabsent = true; - continue; - } - - /* get ptrack bitmap once for all segments of the file */ - if (file->segno == 0) - { - /* release previous value */ - pg_free(ptrack_nonparsed); - ptrack_nonparsed_size = 0; - - ptrack_nonparsed = pg_ptrack_get_and_clear(file->tblspcOid, file->dbOid, - file->relOid, &ptrack_nonparsed_size, backup_conn); - } - - if (ptrack_nonparsed != NULL) - { - /* - * pg_ptrack_get_and_clear() returns ptrack with VARHDR cut out. - * Compute the beginning of the ptrack map related to this segment - * - * HEAPBLOCKS_PER_BYTE. Number of heap pages one ptrack byte can track: 8 - * RELSEG_SIZE. Number of Pages per segment: 131072 - * RELSEG_SIZE/HEAPBLOCKS_PER_BYTE. number of bytes in ptrack file needed - * to keep track on one relsegment: 16384 - */ - start_addr = (RELSEG_SIZE/HEAPBLOCKS_PER_BYTE)*file->segno; - - /* - * If file segment was created after we have read ptrack, - * we won't have a bitmap for this segment. - */ - if (start_addr > ptrack_nonparsed_size) - { - elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); - file->pagemap_isabsent = true; - } - else - { - - if (start_addr + RELSEG_SIZE/HEAPBLOCKS_PER_BYTE > ptrack_nonparsed_size) - { - file->pagemap.bitmapsize = ptrack_nonparsed_size - start_addr; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - else - { - file->pagemap.bitmapsize = RELSEG_SIZE/HEAPBLOCKS_PER_BYTE; - elog(VERBOSE, "pagemap size: %i", file->pagemap.bitmapsize); - } - - file->pagemap.bitmap = pg_malloc(file->pagemap.bitmapsize); - memcpy(file->pagemap.bitmap, ptrack_nonparsed+start_addr, file->pagemap.bitmapsize); - } - } - else - { - /* - * If ptrack file is missing, try to copy the entire file. - * It can happen in two cases: - * - files were created by commands that bypass buffer manager - * and, correspondingly, ptrack mechanism. - * i.e. CREATE DATABASE - * - target relation was deleted. - */ - elog(VERBOSE, "Ptrack is missing for file: %s", file->rel_path); - file->pagemap_isabsent = true; - } - } - } + int ma, mi; + int sscanf_readed_count; + if (sscanf(version_str, "%u.%2u%n", &ma, &mi, &sscanf_readed_count) != 2) + return -1; + if (sscanf_readed_count != strlen(version_str)) + return -1; + return ma * 100 + mi; } /* Check if the instance supports compatible version of ptrack, @@ -148,10 +47,11 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) { PGresult *res_db; char *ptrack_version_str; + int ptrack_version_num; res_db = pgut_execute(backup_conn, "SELECT extnamespace::regnamespace, extversion " - "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'", + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'::name", 0, NULL); if (PQntuples(res_db) > 0) @@ -169,7 +69,7 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) /* ptrack 1.x is supported, save version */ PQclear(res_db); res_db = pgut_execute(backup_conn, - "SELECT proname FROM pg_proc WHERE proname='ptrack_version'", + "SELECT proname FROM pg_catalog.pg_proc WHERE proname='ptrack_version'::name", 0, NULL); if (PQntuples(res_db) == 0) @@ -179,31 +79,41 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) return; } - res_db = pgut_execute(backup_conn, + /* + * it's ok not to have permission to call this old function in PGPRO-11 version (ok_error = true) + * see deprication notice https://postgrespro.com/docs/postgrespro/11/release-pro-11-9-1 + */ + res_db = pgut_execute_extended(backup_conn, "SELECT pg_catalog.ptrack_version()", - 0, NULL); + 0, NULL, true, true); if (PQntuples(res_db) == 0) { - /* TODO: Something went wrong, should we error out here? */ PQclear(res_db); + elog(WARNING, "Can't call pg_catalog.ptrack_version(), it is assumed that there is no ptrack extension installed."); return; } ptrack_version_str = PQgetvalue(res_db, 0, 0); } - if (strcmp(ptrack_version_str, "1.5") == 0) - nodeInfo->ptrack_version_num = 15; - else if (strcmp(ptrack_version_str, "1.6") == 0) - nodeInfo->ptrack_version_num = 16; - else if (strcmp(ptrack_version_str, "1.7") == 0) - nodeInfo->ptrack_version_num = 17; - else if (strcmp(ptrack_version_str, "2.0") == 0) - nodeInfo->ptrack_version_num = 20; - else if (strcmp(ptrack_version_str, "2.1") == 0) - nodeInfo->ptrack_version_num = 21; - else - elog(WARNING, "Update your ptrack to the version 1.5 or upper. Current version is %s", + ptrack_version_num = ptrack_parse_version_string(ptrack_version_str); + if (ptrack_version_num == -1) + /* leave default nodeInfo->ptrack_version_num = 0 from pgNodeInit() */ + elog(WARNING, "Cannot parse ptrack version string \"%s\"", ptrack_version_str); + else + nodeInfo->ptrack_version_num = ptrack_version_num; + + /* ptrack 1.X is buggy, so fall back to DELTA backup strategy for safety */ + if (nodeInfo->ptrack_version_num < 200) + { + if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) + { + elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s. " + "Fall back to DELTA backup.", + ptrack_version_str); + current.backup_mode = BACKUP_MODE_DIFF_DELTA; + } + } PQclear(res_db); } @@ -212,237 +122,29 @@ get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) * Check if ptrack is enabled in target instance */ bool -pg_ptrack_enable(PGconn *backup_conn, int ptrack_version_num) +pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num) { PGresult *res_db; bool result = false; - if (ptrack_version_num < 20) - { - res_db = pgut_execute(backup_conn, "SHOW ptrack_enable", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "on") == 0; - } - else if (ptrack_version_num == 20) - { - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; - } - else + if (ptrack_version_num > 200) { res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; - } - - PQclear(res_db); - return result; -} - - -/* ---------------------------- - * Ptrack 1.* support functions - * ---------------------------- - */ - -/* Clear ptrack files in all databases of the instance we connected to */ -void -pg_ptrack_clear(PGconn *backup_conn, int ptrack_version_num) -{ - PGresult *res_db, - *res; - const char *dbname; - int i; - Oid dbOid, tblspcOid; - char *params[2]; - - // FIXME Perform this check on caller's side - if (ptrack_version_num >= 20) - return; - - params[0] = palloc(64); - params[1] = palloc(64); - res_db = pgut_execute(backup_conn, "SELECT datname, oid, dattablespace FROM pg_database", - 0, NULL); - - for(i = 0; i < PQntuples(res_db); i++) - { - PGconn *tmp_conn; - - dbname = PQgetvalue(res_db, i, 0); - if (strcmp(dbname, "template0") == 0) - continue; - - dbOid = atoi(PQgetvalue(res_db, i, 1)); - tblspcOid = atoi(PQgetvalue(res_db, i, 2)); - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_clear()", - 0, NULL); - PQclear(res); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - PQclear(res); - - pgut_disconnect(tmp_conn); - } - - pfree(params[0]); - pfree(params[1]); - PQclear(res_db); -} - -bool -pg_ptrack_get_and_clear_db(Oid dbOid, Oid tblspcOid, PGconn *backup_conn) -{ - char *params[2]; - char *dbname; - PGresult *res_db; - PGresult *res; - bool result; - - params[0] = palloc(64); - params[1] = palloc(64); - - sprintf(params[0], "%i", dbOid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return false; - - dbname = PQgetvalue(res_db, 0, 0); - - /* Always backup all files from template0 database */ - if (strcmp(dbname, "template0") == 0) - { PQclear(res_db); - return true; } - PQclear(res_db); - - sprintf(params[0], "%i", dbOid); - sprintf(params[1], "%i", tblspcOid); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear_db($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot perform pg_ptrack_get_and_clear_db()"); - - if (!parse_bool(PQgetvalue(res, 0, 0), &result)) - elog(ERROR, - "result of pg_ptrack_get_and_clear_db() is invalid: %s", - PQgetvalue(res, 0, 0)); - - PQclear(res); - pfree(params[0]); - pfree(params[1]); - - return result; -} - -/* Read and clear ptrack files of the target relation. - * Result is a bytea ptrack map of all segments of the target relation. - * case 1: we know a tablespace_oid, db_oid, and rel_filenode - * case 2: we know db_oid and rel_filenode (no tablespace_oid, because file in pg_default) - * case 3: we know only rel_filenode (because file in pg_global) - */ -char * -pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_filenode, - size_t *result_size, PGconn *backup_conn) -{ - PGconn *tmp_conn; - PGresult *res_db, - *res; - char *params[2]; - char *result; - char *val; - - params[0] = palloc(64); - params[1] = palloc(64); - - /* regular file (not in directory 'global') */ - if (db_oid != 0) + else if (ptrack_version_num == 200) { - char *dbname; - - sprintf(params[0], "%i", db_oid); - res_db = pgut_execute(backup_conn, - "SELECT datname FROM pg_database WHERE oid=$1", - 1, (const char **) params); - /* - * If database is not found, it's not an error. - * It could have been deleted since previous backup. - */ - if (PQntuples(res_db) != 1 || PQnfields(res_db) != 1) - return NULL; - - dbname = PQgetvalue(res_db, 0, 0); - - if (strcmp(dbname, "template0") == 0) - { - PQclear(res_db); - return NULL; - } - - tmp_conn = pgut_connect(instance_config.conn_opt.pghost, instance_config.conn_opt.pgport, - dbname, - instance_config.conn_opt.pguser); - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(tmp_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from database \"%s\" by tablespace oid %u and relation oid %u", - dbname, tablespace_oid, rel_filenode); + res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); + result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; PQclear(res_db); - pgut_disconnect(tmp_conn); } - /* file in directory 'global' */ else { - /* - * execute ptrack_get_and_clear for relation in pg_global - * Use backup_conn, cause we can do it from any database. - */ - sprintf(params[0], "%i", tablespace_oid); - sprintf(params[1], "%i", rel_filenode); - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_get_and_clear($1, $2)", - 2, (const char **)params); - - if (PQnfields(res) != 1) - elog(ERROR, "cannot get ptrack file from pg_global tablespace and relation oid %u", - rel_filenode); - } - - val = PQgetvalue(res, 0, 0); - - /* TODO Now pg_ptrack_get_and_clear() returns bytea ending with \x. - * It should be fixed in future ptrack releases, but till then we - * can parse it. - */ - if (strcmp("x", val+1) == 0) - { - /* Ptrack file is missing */ - return NULL; + result = false; } - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - PQclear(res); - pfree(params[0]); - pfree(params[1]); - return result; } @@ -458,20 +160,14 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) uint32 lsn_lo; XLogRecPtr lsn; - if (nodeInfo->ptrack_version_num < 20) - res = pgut_execute(backup_conn, "SELECT pg_catalog.pg_ptrack_control_lsn()", - 0, NULL); - else - { - char query[128]; + char query[128]; - if (nodeInfo->ptrack_version_num == 20) - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); - else - sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); + if (nodeInfo->ptrack_version_num == 200) + sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); + else + sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); - res = pgut_execute(backup_conn, query, 0, NULL); - } + res = pgut_execute(backup_conn, query, 0, NULL); /* Extract timeline and LSN from results of pg_start_backup() */ XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); @@ -482,99 +178,6 @@ get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) return lsn; } -char * -pg_ptrack_get_block(ConnectionArgs *arguments, - Oid dbOid, - Oid tblsOid, - Oid relOid, - BlockNumber blknum, - size_t *result_size, - int ptrack_version_num, - const char *ptrack_schema) -{ - PGresult *res; - char *params[4]; - char *result; - - params[0] = palloc(64); - params[1] = palloc(64); - params[2] = palloc(64); - params[3] = palloc(64); - - /* - * Use tmp_conn, since we may work in parallel threads. - * We can connect to any database. - */ - sprintf(params[0], "%i", tblsOid); - sprintf(params[1], "%i", dbOid); - sprintf(params[2], "%i", relOid); - sprintf(params[3], "%u", blknum); - - if (arguments->conn == NULL) - { - arguments->conn = pgut_connect(instance_config.conn_opt.pghost, - instance_config.conn_opt.pgport, - instance_config.conn_opt.pgdatabase, - instance_config.conn_opt.pguser); - } - - if (arguments->cancel_conn == NULL) - arguments->cancel_conn = PQgetCancel(arguments->conn); - - // elog(LOG, "db %i pg_ptrack_get_block(%i, %i, %u)",dbOid, tblsOid, relOid, blknum); - - if (ptrack_version_num < 20) - res = pgut_execute_parallel(arguments->conn, - arguments->cancel_conn, - "SELECT pg_catalog.pg_ptrack_get_block_2($1, $2, $3, $4)", - 4, (const char **)params, true, false, false); - else - { - char query[128]; - - /* sanity */ - if (!ptrack_schema) - elog(ERROR, "Schema name of ptrack extension is missing"); - - if (ptrack_version_num == 20) - sprintf(query, "SELECT %s.pg_ptrack_get_block($1, $2, $3, $4)", ptrack_schema); - else - elog(ERROR, "ptrack >= 2.1.0 does not support pg_ptrack_get_block()"); - // sprintf(query, "SELECT %s.ptrack_get_block($1, $2, $3, $4)", ptrack_schema); - - res = pgut_execute_parallel(arguments->conn, - arguments->cancel_conn, - query, 4, (const char **)params, - true, false, false); - } - - if (PQnfields(res) != 1) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - if (PQgetisnull(res, 0, 0)) - { - elog(VERBOSE, "cannot get file block for relation oid %u", - relOid); - return NULL; - } - - result = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, 0, 0), - result_size); - - PQclear(res); - - pfree(params[0]); - pfree(params[1]); - pfree(params[2]); - pfree(params[3]); - - return result; -} - /* ---------------------------- * Ptrack 2.* support functions * ---------------------------- @@ -600,7 +203,7 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, if (!ptrack_schema) elog(ERROR, "Schema name of ptrack extension is missing"); - if (ptrack_version_num == 20) + if (ptrack_version_num == 200) sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", ptrack_schema); else @@ -611,7 +214,7 @@ pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, pfree(params[0]); if (PQnfields(res) != 2) - elog(ERROR, "cannot get ptrack pagemapset"); + elog(ERROR, "Cannot get ptrack pagemapset"); /* sanity ? */ diff --git a/src/restore.c b/src/restore.c index 2ade54fa8..f9310dcee 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -39,19 +39,36 @@ typedef struct int ret; } restore_files_arg; -static void create_recovery_conf(time_t backup_id, +static bool control_downloaded = false; +static ControlFileData instance_control; + +static void +print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +static void +print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params); + +#if PG_VERSION_NUM >= 120000 +static void +update_recovery_options(InstanceState *instanceState, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +#else +static void +update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt); +#endif + +static void create_recovery_conf(InstanceState *instanceState, time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params); static void *restore_files(void *arg); static void set_orphan_status(parray *backups, pgBackup *parent_backup); -static void pg12_recovery_config(pgBackup *backup, bool add_include); static void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync); -static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode); + const char *pgdata_path, bool no_sync, bool cleanup_pgdata, + bool backup_has_tblspc); /* * Iterate over backup list to find all ancestors of the broken parent_backup @@ -61,12 +78,8 @@ static void set_orphan_status(parray *backups, pgBackup *parent_backup) { /* chain is intact, but at least one parent is invalid */ - char *parent_backup_id; int j; - /* parent_backup_id is a human-readable backup ID */ - parent_backup_id = base36enc_dup(parent_backup->start_time); - for (j = 0; j < parray_num(backups); j++) { @@ -77,30 +90,30 @@ set_orphan_status(parray *backups, pgBackup *parent_backup) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - parent_backup_id, + backup_id_of(backup), + backup_id_of(parent_backup), status2str(parent_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(backup->start_time), parent_backup_id, + backup_id_of(backup), + backup_id_of(parent_backup), status2str(parent_backup->status)); } } } - pg_free(parent_backup_id); } /* * Entry point of pg_probackup RESTORE and VALIDATE subcommands. */ int -do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, +do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pgRecoveryTarget *rt, pgRestoreParams *params, bool no_sync) { int i = 0; @@ -115,43 +128,107 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, parray *parent_chain = NULL; parray *dbOid_exclude_list = NULL; bool pgdata_is_empty = true; - bool tblspaces_are_empty = true; + bool cleanup_pgdata = false; + bool backup_has_tblspc = true; /* backup contain tablespace */ XLogRecPtr shift_lsn = InvalidXLogRecPtr; + if (instanceState == NULL) + elog(ERROR, "Required parameter not specified: --instance"); + if (params->is_restore) { if (instance_config.pgdata == NULL) - elog(ERROR, - "required parameter not specified: PGDATA (-D, --pgdata)"); + elog(ERROR, "No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)"); + /* Check if restore destination empty */ if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) { + /* if destination directory is empty, then incremental restore may be disabled */ + pgdata_is_empty = false; + /* Check that remote system is NOT running and systemd id is the same as ours */ if (params->incremental_mode != INCR_NONE) { + DestDirIncrCompatibility rc; + const char *message = NULL; + bool ok_to_go = true; + elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", instance_config.pgdata); - check_incremental_compatibility(instance_config.pgdata, - instance_config.system_identifier, - params->incremental_mode); + rc = check_incremental_compatibility(instance_config.pgdata, + instance_config.system_identifier, + params->incremental_mode, + params->partial_db_list, + params->allow_partial_incremental); + if (rc == POSTMASTER_IS_RUNNING) + { + /* Even with force flag it is unwise to run + * incremental restore over running instance + */ + message = "Postmaster is running."; + ok_to_go = false; + } + else if (rc == SYSTEM_ID_MISMATCH) + { + /* + * In force mode it is possible to ignore system id mismatch + * by just wiping clean the destination directory. + */ + if (params->incremental_mode != INCR_NONE && params->force) + cleanup_pgdata = true; + else + { + message = "System ID mismatch."; + ok_to_go = false; + } + } + else if (rc == BACKUP_LABEL_EXISTS) + { + /* + * A big no-no for lsn-based incremental restore + * If there is backup label in PGDATA, then this cluster was probably + * restored from backup, but not started yet. Which means that values + * in pg_control are not synchronized with PGDATA and so we cannot use + * incremental restore in LSN mode, because it is relying on pg_control + * to calculate switchpoint. + */ + if (params->incremental_mode == INCR_LSN) + { + message = "Backup label exists. Cannot use incremental restore in LSN mode."; + ok_to_go = false; + } + } + else if (rc == DEST_IS_NOT_OK) + { + /* + * Something else is wrong. For example, postmaster.pid is mangled, + * so we cannot be sure that postmaster is running or not. + * It is better to just error out. + */ + message = "We cannot be sure about the database state."; + ok_to_go = false; + } else if (rc == PARTIAL_INCREMENTAL_FORBIDDEN) + { + message = "Partial incremental restore into non-empty PGDATA is forbidden."; + ok_to_go = false; + } + + if (!ok_to_go) + elog(ERROR, "Incremental restore is not allowed: %s", message); } else elog(ERROR, "Restore destination is not empty: \"%s\"", instance_config.pgdata); - - /* if destination directory is empty, then incremental restore may be disabled */ - pgdata_is_empty = false; } } - if (instance_name == NULL) - elog(ERROR, "required parameter not specified: --instance"); - elog(LOG, "%s begin.", action); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find backup range we should restore or validate. */ while ((i < parray_num(backups)) && !dest_backup) @@ -180,7 +257,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup->status != BACKUP_STATUS_DONE)) { elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); continue; } @@ -210,10 +287,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, current_backup->status == BACKUP_STATUS_RUNNING) && (!params->no_validate || params->force)) elog(WARNING, "Backup %s has status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); else elog(ERROR, "Backup %s has status: %s", - base36enc(current_backup->start_time), status2str(current_backup->status)); + backup_id_of(current_backup), status2str(current_backup->status)); } if (rt->target_tli) @@ -222,12 +299,16 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, // elog(LOG, "target timeline ID = %u", rt->target_tli); /* Read timeline history files from archives */ - timelines = read_timeline_history(arclog_path, rt->target_tli, true); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, + rt->target_tli, true); - if (!satisfy_timeline(timelines, current_backup)) + if (!timelines) + elog(ERROR, "Failed to get history file for target timeline %i", rt->target_tli); + + if (!satisfy_timeline(timelines, current_backup->tli, current_backup->stop_lsn)) { if (target_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "target backup %s does not satisfy target timeline", + elog(ERROR, "Target backup %s does not satisfy target timeline", base36enc(target_backup_id)); else /* Try to find another backup that satisfies target timeline */ @@ -281,11 +362,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* chain is broken, determine missing backup ID * and orphinize all his descendants */ - char *missing_backup_id; + const char *missing_backup_id; time_t missing_backup_start_time; missing_backup_start_time = tmp_backup->parent_backup; - missing_backup_id = base36enc_dup(tmp_backup->parent_backup); + missing_backup_id = base36enc(tmp_backup->parent_backup); for (j = 0; j < parray_num(backups); j++) { @@ -299,21 +380,20 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(backup->start_time), missing_backup_id); + backup_id_of(backup), missing_backup_id); } else { elog(WARNING, "Backup %s has missing parent %s", - base36enc(backup->start_time), missing_backup_id); + backup_id_of(backup), missing_backup_id); } } } - pg_free(missing_backup_id); /* No point in doing futher */ - elog(ERROR, "%s of backup %s failed.", action, base36enc(dest_backup->start_time)); + elog(ERROR, "%s of backup %s failed.", action, backup_id_of(dest_backup)); } else if (result == ChainIsInvalid) { @@ -324,7 +404,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* sanity */ if (!tmp_backup) elog(ERROR, "Parent full backup for the given backup %s was not found", - base36enc(dest_backup->start_time)); + backup_id_of(dest_backup)); } /* We have found full backup */ @@ -340,9 +420,15 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, */ if (params->is_restore) { - check_tablespace_mapping(dest_backup, params->incremental_mode != INCR_NONE, &tblspaces_are_empty); + int rc = check_tablespace_mapping(dest_backup, + params->incremental_mode != INCR_NONE, params->force, + pgdata_is_empty, params->no_validate); + + /* backup contain no tablespaces */ + if (rc == NoTblspc) + backup_has_tblspc = false; - if (params->incremental_mode != INCR_NONE && pgdata_is_empty && tblspaces_are_empty) + if (params->incremental_mode != INCR_NONE && !cleanup_pgdata && pgdata_is_empty && (rc != NotEmptyTblspc)) { elog(INFO, "Destination directory and tablespace directories are empty, " "disable incremental restore"); @@ -350,6 +436,9 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, } /* no point in checking external directories if their restore is not requested */ + //TODO: + // - make check_external_dir_mapping more like check_tablespace_mapping + // - honor force flag in case of incremental restore just like check_tablespace_mapping if (!params->skip_external_dirs) check_external_dir_mapping(dest_backup, params->incremental_mode != INCR_NONE); } @@ -409,13 +498,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { RedoParams redo; parray *timelines = NULL; - get_redo(instance_config.pgdata, &redo); + get_redo(instance_config.pgdata, FIO_DB_HOST, &redo); if (redo.checksum_version == 0) elog(ERROR, "Incremental restore in 'lsn' mode require " "data_checksums to be enabled in destination data directory"); + if (!control_downloaded) + get_control_file_or_back_file(instance_config.pgdata, FIO_DB_HOST, + &instance_control); - timelines = read_timeline_history(arclog_path, redo.tli, false); + timelines = read_timeline_history(instanceState->instance_wal_subdir_path, + redo.tli, false); if (!timelines) elog(WARNING, "Failed to get history for redo timeline %i, " @@ -434,7 +527,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (redo.tli == tmp_backup->tli) { elog(INFO, "Backup %s is chosen as shiftpoint, its Stop LSN will be used as shift LSN", - base36enc(tmp_backup->start_time)); + backup_id_of(tmp_backup)); shift_lsn = tmp_backup->stop_lsn; break; @@ -458,7 +551,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, else elog(INFO, "Backup %s cannot be a shiftpoint, " "because its tli %i is not in history of redo timeline %i", - base36enc(tmp_backup->start_time), tmp_backup->tli, redo.tli); + backup_id_of(tmp_backup), tmp_backup->tli, redo.tli); } tmp_backup = tmp_backup->parent_backup_link; @@ -467,27 +560,27 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (XLogRecPtrIsInvalid(shift_lsn)) elog(ERROR, "Cannot perform incremental restore of backup chain %s in 'lsn' mode, " "because destination directory redo point %X/%X on tli %i is out of reach", - base36enc(dest_backup->start_time), + backup_id_of(dest_backup), (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); else elog(INFO, "Destination directory redo point %X/%X on tli %i is " "within reach of backup %s with Stop LSN %X/%X on tli %i", (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, - base36enc(tmp_backup->start_time), + backup_id_of(tmp_backup), (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, tmp_backup->tli); elog(INFO, "shift LSN: %X/%X", (uint32) (shift_lsn >> 32), (uint32) shift_lsn); - params->shift_lsn = shift_lsn; } + params->shift_lsn = shift_lsn; /* for validation or restore with enabled validation */ if (!params->is_restore || !params->no_validate) { if (dest_backup->backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Validating parents for backup %s", base36enc(dest_backup->start_time)); + elog(INFO, "Validating parents for backup %s", backup_id_of(dest_backup)); /* * Validate backups from base_full_backup to dest_backup. @@ -496,18 +589,11 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, { tmp_backup = (pgBackup *) parray_get(parent_chain, i); - /* Do not interrupt, validate the next backup */ - if (!lock_backup(tmp_backup, true)) + /* lock every backup in chain in read-only mode */ + if (!lock_backup(tmp_backup, true, false)) { - if (params->is_restore) - elog(ERROR, "Cannot lock backup %s directory", - base36enc(tmp_backup->start_time)); - else - { - elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(tmp_backup->start_time)); - continue; - } + elog(ERROR, "Cannot lock backup %s directory", + backup_id_of(tmp_backup)); } /* validate datafiles only */ @@ -537,7 +623,7 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, * We pass base_full_backup timeline as last argument to this function, * because it's needed to form the name of xlog file. */ - validate_wal(dest_backup, arclog_path, rt->target_time, + validate_wal(dest_backup, instanceState->instance_wal_subdir_path, rt->target_time, rt->target_xid, rt->target_lsn, dest_backup->tli, instance_config.xlog_seg_size); } @@ -554,27 +640,27 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, dest_backup->status == BACKUP_STATUS_DONE) { if (params->no_validate) - elog(WARNING, "Backup %s is used without validation.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is used without validation.", backup_id_of(dest_backup)); else - elog(INFO, "Backup %s is valid.", base36enc(dest_backup->start_time)); + elog(INFO, "Backup %s is valid.", backup_id_of(dest_backup)); } else if (dest_backup->status == BACKUP_STATUS_CORRUPT) { if (params->force) - elog(WARNING, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is corrupt.", backup_id_of(dest_backup)); else - elog(ERROR, "Backup %s is corrupt.", base36enc(dest_backup->start_time)); + elog(ERROR, "Backup %s is corrupt.", backup_id_of(dest_backup)); } else if (dest_backup->status == BACKUP_STATUS_ORPHAN) { if (params->force) - elog(WARNING, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + elog(WARNING, "Backup %s is orphan.", backup_id_of(dest_backup)); else - elog(ERROR, "Backup %s is orphan.", base36enc(dest_backup->start_time)); + elog(ERROR, "Backup %s is orphan.", backup_id_of(dest_backup)); } else elog(ERROR, "Backup %s has status: %s", - base36enc(dest_backup->start_time), status2str(dest_backup->status)); + backup_id_of(dest_backup), status2str(dest_backup->status)); /* We ensured that all backups are valid, now restore if required */ @@ -598,21 +684,27 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, if (rt->lsn_string && parse_server_version(dest_backup->server_version) < 100000) elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", - base36enc(dest_backup->start_time), + backup_id_of(dest_backup), dest_backup->server_version); - restore_chain(dest_backup, parent_chain, dbOid_exclude_list, - params, instance_config.pgdata, no_sync); + if (instance_config.remote.host) + elog(INFO, "Restoring the database from backup %s on %s", backup_id_of(dest_backup), instance_config.remote.host); + else + elog(INFO, "Restoring the database from backup %s", backup_id_of(dest_backup)); + + restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, + instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); + //TODO rename and update comment /* Create recovery.conf with given recovery target parameters */ - create_recovery_conf(target_backup_id, rt, dest_backup, params); + create_recovery_conf(instanceState, target_backup_id, rt, dest_backup, params); } /* ssh connection to longer needed */ fio_disconnect(); elog(INFO, "%s of backup %s completed.", - action, base36enc(dest_backup->start_time)); + action, backup_id_of(dest_backup)); /* cleanup */ parray_walk(backups, pgBackupFree); @@ -624,17 +716,22 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt, /* * Restore backup chain. + * Flag 'cleanup_pgdata' demands the removing of already existing content in PGDATA. */ void restore_chain(pgBackup *dest_backup, parray *parent_chain, parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync) + const char *pgdata_path, bool no_sync, bool cleanup_pgdata, + bool backup_has_tblspc) { int i; - char timestamp[100]; parray *pgdata_files = NULL; parray *dest_files = NULL; parray *external_dirs = NULL; + pgFile *dest_pg_control_file = NULL; + char dest_pg_control_fullpath[MAXPGPATH]; + char dest_pg_control_bak_fullpath[MAXPGPATH]; + /* arrays with meta info for multi threaded backup */ pthread_t *threads; restore_files_arg *threads_args; @@ -650,9 +747,6 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time_t start_time, end_time; /* Preparations for actual restoring */ - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time); - elog(INFO, "Restoring the database from backup at %s", timestamp); - dest_files = get_backup_filelist(dest_backup, true); /* Lock backup chain and make sanity checks */ @@ -660,18 +754,18 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - if (!lock_backup(backup, true)) - elog(ERROR, "Cannot lock backup %s", base36enc(backup->start_time)); + if (!lock_backup(backup, true, false)) + elog(ERROR, "Cannot lock backup %s", backup_id_of(backup)); if (backup->status != BACKUP_STATUS_OK && backup->status != BACKUP_STATUS_DONE) { if (params->force) elog(WARNING, "Backup %s is not valid, restore is forced", - base36enc(backup->start_time)); + backup_id_of(backup)); else elog(ERROR, "Backup %s cannot be restored because it is not valid", - base36enc(backup->start_time)); + backup_id_of(backup)); } /* confirm block size compatibility */ @@ -708,7 +802,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, use_bitmap = false; if (params->incremental_mode != INCR_NONE) - elog(ERROR, "incremental restore is not possible for backups older than 2.3.0 version"); + elog(ERROR, "Incremental restore is not possible for backups older than 2.3.0 version"); } /* There is no point in bitmap restore, when restoring a single FULL backup, @@ -726,9 +820,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, * Restore dest_backup internal directories. */ create_data_directories(dest_files, instance_config.pgdata, - dest_backup->root_dir, true, + dest_backup->root_dir, backup_has_tblspc, params->incremental_mode != INCR_NONE, - FIO_DB_HOST); + FIO_DB_HOST, params->waldir); /* * Restore dest_backup external directories. @@ -749,7 +843,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } /* - * Setup directory structure for external directories and file locks + * Setup directory structure for external directories */ for (i = 0; i < parray_num(dest_files); i++) { @@ -770,27 +864,33 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, external_path = parray_get(external_dirs, file->external_dir_num - 1); join_path_components(dirpath, external_path, file->rel_path); - elog(VERBOSE, "Create external directory \"%s\"", dirpath); + elog(LOG, "Create external directory \"%s\"", dirpath); fio_mkdir(dirpath, file->mode, FIO_DB_HOST); } - - /* setup threads */ - pg_atomic_clear_flag(&file->lock); } + /* setup threads */ + pfilearray_clear_locks(dest_files); + /* Get list of files in destination directory and remove redundant files */ - if (params->incremental_mode != INCR_NONE) + if (params->incremental_mode != INCR_NONE || cleanup_pgdata) { pgdata_files = parray_new(); elog(INFO, "Extracting the content of destination directory for incremental restore"); time(&start_time); - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); - else - dir_list_file(pgdata_files, pgdata_path, - false, true, false, false, true, 0, FIO_LOCAL_HOST); + fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); + + /* + * TODO: + * 1. Currently we are cleaning the tablespaces in check_tablespace_mapping and PGDATA here. + * It would be great to do all this work in one place. + * + * 2. In case of tablespace remapping we do not cleanup the old tablespace path, + * it is just left as it is. + * Lookup tests.incr_restore.IncrRestoreTest.test_incr_restore_with_tablespace_5 + */ /* get external dirs content */ if (external_dirs) @@ -800,13 +900,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, char *external_path = parray_get(external_dirs, i); parray *external_files = parray_new(); - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(external_files, external_path, - false, true, false, false, true, i+1); - else - dir_list_file(external_files, external_path, - false, true, false, false, true, i+1, - FIO_LOCAL_HOST); + fio_list_dir(external_files, external_path, + false, true, false, false, true, i+1); parray_concat(pgdata_files, external_files); parray_free(external_files); @@ -826,25 +921,51 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, time(&start_time); for (i = 0; i < parray_num(pgdata_files); i++) { - pgFile *file = (pgFile *) parray_get(pgdata_files, i); + bool redundant = true; + pgFile *file = (pgFile *) parray_get(pgdata_files, i); + + if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal)) + redundant = false; + + /* pg_filenode.map are always restored, because it's crc cannot be trusted */ + if (file->external_dir_num == 0 && + pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) + redundant = true; + + /* global/pg_control.pbk.bak are always keeped, because it's needed for restart failed incremental restore */ + if (file->external_dir_num == 0 && + pg_strcasecmp(file->rel_path, XLOG_CONTROL_BAK_FILE) == 0) + redundant = false; + + /* do not delete the useful internal directories */ + if (S_ISDIR(file->mode) && !redundant) + continue; /* if file does not exists in destination list, then we can safely unlink it */ - if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal) == NULL) + if (cleanup_pgdata || redundant) { char fullpath[MAXPGPATH]; join_path_components(fullpath, pgdata_path, file->rel_path); -// fio_pgFileDelete(file, full_file_path); fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + elog(LOG, "Deleted file \"%s\"", fullpath); /* shrink pgdata list */ + pgFileFree(file); parray_remove(pgdata_files, i); i--; } } + if (cleanup_pgdata) + { + /* Destination PGDATA and tablespaces were cleaned up, so it's the regular restore from this point */ + params->incremental_mode = INCR_NONE; + parray_free(pgdata_files); + pgdata_files = NULL; + } + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -868,6 +989,42 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, dest_bytes = dest_backup->pgdata_bytes; pretty_size(dest_bytes, pretty_dest_bytes, lengthof(pretty_dest_bytes)); + /* + * [Issue #313] + * find pg_control file (in already sorted earlier dest_files, see parray_qsort(backup->files...)) + * and exclude it from list for future special processing + */ + { + int control_file_elem_index; + pgFile search_key; + MemSet(&search_key, 0, sizeof(pgFile)); + /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ + search_key.rel_path = XLOG_CONTROL_FILE; + search_key.external_dir_num = 0; + control_file_elem_index = parray_bsearch_index(dest_files, &search_key, pgFileCompareRelPathWithExternal); + + if (control_file_elem_index < 0) + elog(ERROR, "File \"%s\" not found in backup %s", XLOG_CONTROL_FILE, base36enc(dest_backup->start_time)); + dest_pg_control_file = (pgFile *) parray_get(dest_files, control_file_elem_index); + parray_remove(dest_files, control_file_elem_index); + + join_path_components(dest_pg_control_fullpath, pgdata_path, XLOG_CONTROL_FILE); + join_path_components(dest_pg_control_bak_fullpath, pgdata_path, XLOG_CONTROL_BAK_FILE); + /* + * rename (if it exist) dest control file before restoring + * if it doesn't exist, that mean, that we already restoring in a previously failed + * pgdata, where XLOG_CONTROL_BAK_FILE exist + */ + if (params->incremental_mode != INCR_NONE) + { + if (fio_access(dest_pg_control_fullpath,F_OK,FIO_DB_HOST) == 0){ + if (fio_rename(dest_pg_control_fullpath, dest_pg_control_bak_fullpath, FIO_DB_HOST) < 0) + elog(WARNING, "Cannot rename file \"%s\" to \"%s\": %s", + dest_pg_control_fullpath, dest_pg_control_bak_fullpath, strerror(errno)); + } + } + } + elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); time(&start_time); thread_interrupted = false; @@ -908,6 +1065,32 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, total_bytes += threads_args[i].restored_bytes; } + /* [Issue #313] copy pg_control at very end */ + if (restore_isok) + { + FILE *out = NULL; + elog(progress ? INFO : LOG, "Progress: Restore file \"%s\"", + dest_pg_control_file->rel_path); + + out = fio_fopen(dest_pg_control_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + + total_bytes += restore_non_data_file(parent_chain, + dest_backup, + dest_pg_control_file, + out, + dest_pg_control_fullpath, false); + fio_fclose(out); + /* Now backup control file can be deleted */ + if (params->incremental_mode != INCR_NONE) + { + pgFile *dst_control; + dst_control = pgFileNew(dest_pg_control_bak_fullpath, XLOG_CONTROL_BAK_FILE, + true,0, FIO_BACKUP_HOST); + fio_delete(dst_control->mode, dest_pg_control_bak_fullpath, FIO_LOCAL_HOST); + pgFileFree(dst_control); + } + } + time(&end_time); pretty_time_interval(difftime(end_time, start_time), pretty_time, lengthof(pretty_time)); @@ -992,6 +1175,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, parray_free(pgdata_files); } + if(dest_pg_control_file) pgFileFree(dest_pg_control_file); + for (i = parray_num(parent_chain) - 1; i >= 0; i--) { pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); @@ -1022,6 +1207,7 @@ restore_files(void *arg) bool already_exists = false; PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ + char *errmsg = NULL; /* remote agent error message */ pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); /* Directories were created before */ @@ -1035,9 +1221,8 @@ restore_files(void *arg) if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during restore"); - if (progress) - elog(INFO, "Progress: (%d/%lu). Restore file \"%s\"", - i + 1, n_files, dest_file->rel_path); + elog(progress ? INFO : LOG, "Progress: (%d/%lu). Restore file \"%s\"", + i + 1, n_files, dest_file->rel_path); /* Only files from pgdata can be skipped by partial restore */ if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) @@ -1053,7 +1238,7 @@ restore_files(void *arg) create_empty_file(FIO_BACKUP_HOST, arguments->to_root, FIO_DB_HOST, dest_file); - elog(VERBOSE, "Skip file due to partial restore: \"%s\"", + elog(LOG, "Skip file due to partial restore: \"%s\"", dest_file->rel_path); continue; } @@ -1063,7 +1248,7 @@ restore_files(void *arg) if ((dest_file->external_dir_num == 0) && strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) { - elog(VERBOSE, "Skip tablespace_map"); + elog(LOG, "Skip tablespace_map"); continue; } @@ -1071,7 +1256,7 @@ restore_files(void *arg) if ((dest_file->external_dir_num == 0) && strcmp(DATABASE_MAP, dest_file->rel_path) == 0) { - elog(VERBOSE, "Skip database_map"); + elog(LOG, "Skip database_map"); continue; } @@ -1143,9 +1328,9 @@ restore_files(void *arg) strerror(errno)); if (!dest_file->is_datafile || dest_file->is_cfs) - elog(VERBOSE, "Restoring nonedata file: \"%s\"", to_fullpath); + elog(LOG, "Restoring non-data file: \"%s\"", to_fullpath); else - elog(VERBOSE, "Restoring data file: \"%s\"", to_fullpath); + elog(LOG, "Restoring data file: \"%s\"", to_fullpath); // If destination file is 0 sized, then just close it and go for the next if (dest_file->write_size == 0) @@ -1165,16 +1350,20 @@ restore_files(void *arg) } else { - /* disable stdio buffering for local destination nonedata file */ + /* disable stdio buffering for local destination non-data file */ if (!fio_is_remote_file(out)) setvbuf(out, NULL, _IONBF, BUFSIZ); - /* Destination file is nonedata file */ + /* Destination file is non-data file */ arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, arguments->dest_backup, dest_file, out, to_fullpath, already_exists); } done: + /* Writing is asynchronous in case of restore in remote mode, so check the agent status */ + if (fio_check_error_file(out, &errmsg)) + elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath, errmsg); + /* close file */ if (fio_fclose(out) != 0) elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, @@ -1202,22 +1391,18 @@ restore_files(void *arg) } /* - * Create recovery.conf (probackup_recovery.conf in case of PG12) + * Create recovery.conf (postgresql.auto.conf in case of PG12) * with given recovery target parameters */ static void -create_recovery_conf(time_t backup_id, +create_recovery_conf(InstanceState *instanceState, time_t backup_id, pgRecoveryTarget *rt, pgBackup *backup, pgRestoreParams *params) { - char path[MAXPGPATH]; - FILE *fp; - bool pitr_requested; bool target_latest; bool target_immediate; bool restore_command_provided = false; - char restore_command_guc[16384]; if (instance_config.restore_command && (pg_strcasecmp(instance_config.restore_command, "none") != 0)) @@ -1226,8 +1411,10 @@ create_recovery_conf(time_t backup_id, } /* restore-target='latest' support */ - target_latest = rt->target_stop != NULL && - strcmp(rt->target_stop, "latest") == 0; + target_latest = (rt->target_tli_string != NULL && + strcmp(rt->target_tli_string, "latest") == 0) || + (rt->target_stop != NULL && + strcmp(rt->target_stop, "latest") == 0); target_immediate = rt->target_stop != NULL && strcmp(rt->target_stop, "immediate") == 0; @@ -1249,113 +1436,101 @@ create_recovery_conf(time_t backup_id, * We will get a replica that is "in the future" to the master. * We accept this risk because its probability is low. */ - pitr_requested = !backup->stream || rt->time_string || + if (!backup->stream || rt->time_string || rt->xid_string || rt->lsn_string || rt->target_name || - target_immediate || target_latest || restore_command_provided; - - /* No need to generate recovery.conf at all. */ - if (!(pitr_requested || params->restore_as_replica)) - { - /* - * Restoring STREAM backup without PITR and not as replica, - * recovery.signal and standby.signal for PG12 are not needed - * - * We do not add "include" option in this case because - * here we are creating empty "probackup_recovery.conf" - * to handle possible already existing "include" - * directive pointing to "probackup_recovery.conf". - * If don`t do that, recovery will fail. - */ - pg12_recovery_config(backup, false); - return; - } + target_immediate || target_latest || restore_command_provided) + params->recovery_settings_mode = PITR_REQUESTED; + /* + * The recovery-target-timeline option can be 'latest' for streaming backups. + * This operation requires a WAL archive for PITR. + */ + if (rt->target_tli && backup->stream && params->recovery_settings_mode != PITR_REQUESTED) + elog(WARNING, "The '--recovery-target-timeline' option applied for STREAM backup. " + "The timeline number will be ignored."); elog(LOG, "----------------------------------------"); + #if PG_VERSION_NUM >= 120000 - elog(LOG, "creating probackup_recovery.conf"); - pg12_recovery_config(backup, true); - snprintf(path, lengthof(path), "%s/probackup_recovery.conf", instance_config.pgdata); + update_recovery_options(instanceState, backup, params, rt); #else - elog(LOG, "creating recovery.conf"); - snprintf(path, lengthof(path), "%s/recovery.conf", instance_config.pgdata); + update_recovery_options_before_v12(instanceState, backup, params, rt); #endif +} - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); -#if PG_VERSION_NUM >= 120000 - fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); -#else - fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); -#endif +/* TODO get rid of using global variables: instance_config */ +static void +print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) +{ + char restore_command_guc[16384]; + fio_fprintf(fp, "## recovery settings\n"); - /* construct restore_command */ - if (pitr_requested) + /* If restore_command is provided, use it. Otherwise construct it from scratch. */ + if (instance_config.restore_command && + (pg_strcasecmp(instance_config.restore_command, "none") != 0)) + sprintf(restore_command_guc, "%s", instance_config.restore_command); + else { - fio_fprintf(fp, "\n## recovery settings\n"); - /* If restore_command is provided, use it. Otherwise construct it from scratch. */ - if (restore_command_provided) - sprintf(restore_command_guc, "%s", instance_config.restore_command); - else + /* default cmdline, ok for local restore */ + sprintf(restore_command_guc, "\"%s\" archive-get -B \"%s\" --instance \"%s\" " + "--wal-file-path=%%p --wal-file-name=%%f", + PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, + /* TODO What is going on here? Why do we use catalog path as wal-file-path? */ + instanceState->catalog_state->catalog_path, instanceState->instance_name); + + /* append --remote-* parameters provided via --archive-* settings */ + if (instance_config.archive.host) { - /* default cmdline, ok for local restore */ - sprintf(restore_command_guc, "%s archive-get -B %s --instance %s " - "--wal-file-path=%%p --wal-file-name=%%f", - PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, - backup_path, instance_name); - - /* append --remote-* parameters provided via --archive-* settings */ - if (instance_config.archive.host) - { - strcat(restore_command_guc, " --remote-host="); - strcat(restore_command_guc, instance_config.archive.host); - } + strcat(restore_command_guc, " --remote-host="); + strcat(restore_command_guc, instance_config.archive.host); + } - if (instance_config.archive.port) - { - strcat(restore_command_guc, " --remote-port="); - strcat(restore_command_guc, instance_config.archive.port); - } + if (instance_config.archive.port) + { + strcat(restore_command_guc, " --remote-port="); + strcat(restore_command_guc, instance_config.archive.port); + } - if (instance_config.archive.user) - { - strcat(restore_command_guc, " --remote-user="); - strcat(restore_command_guc, instance_config.archive.user); - } + if (instance_config.archive.user) + { + strcat(restore_command_guc, " --remote-user="); + strcat(restore_command_guc, instance_config.archive.user); } + } - /* - * We've already checked that only one of the four following mutually - * exclusive options is specified, so the order of calls is insignificant. - */ - if (rt->target_name) - fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); + /* + * We've already checked that only one of the four following mutually + * exclusive options is specified, so the order of calls is insignificant. + */ + if (rt->target_name) + fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); - if (rt->time_string) - fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); + if (rt->time_string) + fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); - if (rt->xid_string) - fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); + if (rt->xid_string) + fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); - if (rt->lsn_string) - fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); + if (rt->lsn_string) + fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - if (rt->target_stop && target_immediate) - fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); + if (rt->target_stop && (strcmp(rt->target_stop, "immediate") == 0)) + fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); - if (rt->inclusive_specified) - fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", - rt->target_inclusive ? "true" : "false"); + if (rt->inclusive_specified) + fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", + rt->target_inclusive ? "true" : "false"); - if (rt->target_tli) - fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + if (rt->target_tli) + fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); + else + { + if (rt->target_tli_string) + fio_fprintf(fp, "recovery_target_timeline = '%s'\n", rt->target_tli_string); + else if (rt->target_stop && (strcmp(rt->target_stop, "latest") == 0)) + fio_fprintf(fp, "recovery_target_timeline = 'latest'\n"); +#if PG_VERSION_NUM >= 120000 else { /* @@ -1363,151 +1538,251 @@ create_recovery_conf(time_t backup_id, * is extremely risky. Explicitly preserve old behavior of recovering to current * timneline for PG12. */ -#if PG_VERSION_NUM >= 120000 fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); -#endif } - - if (rt->target_action) - fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); - else - /* default recovery_target_action is 'pause' */ - fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); +#endif } - if (pitr_requested) + if (rt->target_action) + fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); + else + /* default recovery_target_action is 'pause' */ + fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); + + elog(LOG, "Setting restore_command to '%s'", restore_command_guc); + fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); +} + +static void +print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params) +{ + fio_fprintf(fp, "\n## standby settings\n"); + if (params->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); + else if (backup->primary_conninfo) + fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); + + if (params->primary_slot_name != NULL) + fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); +} + +#if PG_VERSION_NUM < 120000 +static void +update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) +{ + FILE *fp; + char path[MAXPGPATH]; + + /* + * If PITR is not requested and instance is not restored as replica, + * then recovery.conf should not be created. + */ + if (params->recovery_settings_mode != PITR_REQUESTED && + !params->restore_as_replica) { - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); + return; } + elog(LOG, "update recovery settings in recovery.conf"); + join_path_components(path, instance_config.pgdata, "recovery.conf"); + + fp = fio_fopen(path, "w", FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "Cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); + + fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", + PROGRAM_VERSION); + + if (params->recovery_settings_mode == PITR_REQUESTED) + print_recovery_settings(instanceState, fp, backup, params, rt); + if (params->restore_as_replica) { - fio_fprintf(fp, "\n## standby settings\n"); - /* standby_mode was removed in PG12 */ -#if PG_VERSION_NUM < 120000 + print_standby_settings_common(fp, backup, params); fio_fprintf(fp, "standby_mode = 'on'\n"); -#endif - - if (params->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); - else if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); - - if (params->primary_slot_name != NULL) - fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); } if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(ERROR, "Cannot write file \"%s\": %s", path, strerror(errno)); +} +#endif +/* + * Read postgresql.auto.conf, clean old recovery options, + * to avoid unexpected intersections. + * Write recovery options for this backup. + */ #if PG_VERSION_NUM >= 120000 - /* - * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. - * In older versions presense of recovery.conf alone was enough. - * To keep behaviour consistent with older versions, - * we are forced to create "recovery.signal" - * even when only restore_command is provided. - * Presense of "recovery.signal" by itself determine only - * one thing: do PostgreSQL must switch to a new timeline - * after successfull recovery or not? - */ - if (pitr_requested) - { - elog(LOG, "creating recovery.signal file"); - snprintf(path, lengthof(path), "%s/recovery.signal", instance_config.pgdata); +static void +update_recovery_options(InstanceState *instanceState, pgBackup *backup, + pgRestoreParams *params, pgRecoveryTarget *rt) - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); +{ + char postgres_auto_path[MAXPGPATH]; + char postgres_auto_path_tmp[MAXPGPATH]; + char path[MAXPGPATH]; + FILE *fp = NULL; + FILE *fp_tmp = NULL; + struct stat st; + char current_time_str[100]; + /* postgresql.auto.conf parsing */ + char line[16384] = "\0"; + char *buf = NULL; + int buf_len = 0; + int buf_len_max = 16384; - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, + elog(LOG, "update recovery settings in postgresql.auto.conf"); + + time2iso(current_time_str, lengthof(current_time_str), current_time, false); + + join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); + + if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) + { + /* file not found is not an error case */ + if (errno != ENOENT) + elog(ERROR, "Cannot stat file \"%s\": %s", postgres_auto_path, strerror(errno)); + st.st_size = 0; } - if (params->restore_as_replica) + /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ + if (st.st_size > 0) { - elog(LOG, "creating standby.signal file"); - snprintf(path, lengthof(path), "%s/standby.signal", instance_config.pgdata); - - fp = fio_fopen(path, "w", FIO_DB_HOST); + fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write file \"%s\": %s", path, - strerror(errno)); + elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); } -#endif -} -/* - * Create empty probackup_recovery.conf in PGDATA and - * add "include" directive to postgresql.auto.conf + sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); + fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); + if (fp_tmp == NULL) + elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); - * When restoring PG12 we always(!) must do this, even - * when restoring STREAM backup without PITR or replica options - * because restored instance may have been previously backed up - * and restored again and user didn`t cleaned up postgresql.auto.conf. + while (fp && fgets(line, lengthof(line), fp)) + { + /* ignore "include 'probackup_recovery.conf'" directive */ + if (strstr(line, "include") && + strstr(line, "probackup_recovery.conf")) + { + continue; + } - * So for recovery to work regardless of all this factors - * we must always create empty probackup_recovery.conf file. - */ -static void -pg12_recovery_config(pgBackup *backup, bool add_include) -{ -#if PG_VERSION_NUM >= 120000 - char probackup_recovery_path[MAXPGPATH]; - char postgres_auto_path[MAXPGPATH]; - FILE *fp; + /* ignore already existing recovery options */ + if (strstr(line, "restore_command") || + strstr(line, "recovery_target")) + { + continue; + } - if (add_include) - { - char current_time_str[100]; + if (!buf) + buf = pgut_malloc(buf_len_max); - time2iso(current_time_str, lengthof(current_time_str), current_time); + /* avoid buffer overflow */ + if ((buf_len + strlen(line)) >= buf_len_max) + { + buf_len_max += (buf_len + strlen(line)) *2; + buf = pgut_realloc(buf, buf_len_max); + } - snprintf(postgres_auto_path, lengthof(postgres_auto_path), - "%s/postgresql.auto.conf", instance_config.pgdata); + buf_len += snprintf(buf+buf_len, sizeof(line), "%s", line); + } + + /* close input postgresql.auto.conf */ + if (fp) + fio_close_stream(fp); + /* Write data to postgresql.auto.conf.tmp */ + if (buf_len > 0 && + (fio_fwrite(fp_tmp, buf, buf_len) != buf_len)) + elog(ERROR, "Cannot write to \"%s\": %s", + postgres_auto_path_tmp, strerror(errno)); + + if (fio_fflush(fp_tmp) != 0 || + fio_fclose(fp_tmp)) + elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path_tmp, + strerror(errno)); + pg_free(buf); + + if (fio_rename(postgres_auto_path_tmp, postgres_auto_path, FIO_DB_HOST) < 0) + elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", + postgres_auto_path_tmp, postgres_auto_path, strerror(errno)); + + if (fio_chmod(postgres_auto_path, FILE_PERMISSION, FIO_DB_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", postgres_auto_path, strerror(errno)); + + if (params) + { fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); if (fp == NULL) - elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + elog(ERROR, "Cannot open file \"%s\": %s", postgres_auto_path, strerror(errno)); - // TODO: check if include 'probackup_recovery.conf' already exists - fio_fprintf(fp, "\n# created by pg_probackup restore of backup %s at '%s'\n", - base36enc(backup->start_time), current_time_str); - fio_fprintf(fp, "include '%s'\n", "probackup_recovery.conf"); + fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", + backup_id_of(backup), current_time_str); + + if (params->recovery_settings_mode == PITR_REQUESTED) + print_recovery_settings(instanceState, fp, backup, params, rt); + + if (params->restore_as_replica) + print_standby_settings_common(fp, backup, params); if (fio_fflush(fp) != 0 || fio_fclose(fp)) - elog(ERROR, "cannot write to file \"%s\": %s", postgres_auto_path, + elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path, strerror(errno)); - } - /* Create empty probackup_recovery.conf */ - snprintf(probackup_recovery_path, lengthof(probackup_recovery_path), - "%s/probackup_recovery.conf", instance_config.pgdata); - fp = fio_fopen(probackup_recovery_path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "cannot open file \"%s\": %s", probackup_recovery_path, - strerror(errno)); + /* + * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. + * In older versions presense of recovery.conf alone was enough. + * To keep behaviour consistent with older versions, + * we are forced to create "recovery.signal" + * even when only restore_command is provided. + * Presense of "recovery.signal" by itself determine only + * one thing: do PostgreSQL must switch to a new timeline + * after successfull recovery or not? + */ + if (params->recovery_settings_mode == PITR_REQUESTED) + { + elog(LOG, "creating recovery.signal file"); + join_path_components(path, instance_config.pgdata, "recovery.signal"); - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "cannot write to file \"%s\": %s", probackup_recovery_path, - strerror(errno)); -#endif - return; + fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "Cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "Cannot write file \"%s\": %s", path, + strerror(errno)); + } + + if (params->restore_as_replica) + { + elog(LOG, "creating standby.signal file"); + join_path_components(path, instance_config.pgdata, "standby.signal"); + + fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + if (fp == NULL) + elog(ERROR, "Cannot open file \"%s\": %s", path, + strerror(errno)); + + if (fio_fflush(fp) != 0 || + fio_fclose(fp)) + elog(ERROR, "Cannot write file \"%s\": %s", path, + strerror(errno)); + } + } } +#endif /* * Try to read a timeline's history file. @@ -1539,12 +1814,12 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict if (fd == NULL) { if (errno != ENOENT) - elog(ERROR, "could not open file \"%s\": %s", path, + elog(ERROR, "Could not open file \"%s\": %s", path, strerror(errno)); /* There is no history file for target timeline */ if (strict) - elog(ERROR, "recovery target timeline %u does not exist", + elog(ERROR, "Recovery target timeline %u does not exist", targetTLI); else return NULL; @@ -1578,12 +1853,12 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict { /* expect a numeric timeline ID as first field of line */ elog(ERROR, - "syntax error in history file: %s. Expected a numeric timeline ID.", + "Syntax error in history file: %s. Expected a numeric timeline ID.", fline); } if (nfields != 3) elog(ERROR, - "syntax error in history file: %s. Expected a transaction log switchpoint location.", + "Syntax error in history file: %s. Expected a transaction log switchpoint location.", fline); if (last_timeline && tli <= last_timeline->tli) @@ -1602,7 +1877,7 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict } if (fd && (ferror(fd))) - elog(ERROR, "Failed to read from file: \"%s\"", path); + elog(ERROR, "Failed to read from file: \"%s\"", path); if (fd) fclose(fd); @@ -1610,6 +1885,14 @@ read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict if (last_timeline && targetTLI <= last_timeline->tli) elog(ERROR, "Timeline IDs must be less than child timeline's ID."); + /* History file is empty or corrupted */ + if (parray_num(result) == 0 && targetTLI != 1) + { + elog(WARNING, "History file is corrupted or missing: \"%s\"", path); + pg_free(result); + return NULL; + } + /* append target timeline */ entry = pgut_new(TimeLineHistoryEntry); entry->tli = targetTLI; @@ -1638,18 +1921,22 @@ satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) /* TODO description */ bool -satisfy_timeline(const parray *timelines, const pgBackup *backup) +satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn) { int i; + elog(VERBOSE, "satisfy_timeline() checking: tli = %X, lsn = %X/%X", + tli, (uint32) (lsn >> 32), (uint32) lsn); for (i = 0; i < parray_num(timelines); i++) { TimeLineHistoryEntry *timeline; timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); - if (backup->tli == timeline->tli && + elog(VERBOSE, "satisfy_timeline() check %i entry: timeline->tli = %X, timeline->end = %X/%X", + i, timeline->tli, (uint32) (timeline->end >> 32), (uint32) timeline->end); + if (tli == timeline->tli && (XLogRecPtrIsInvalid(timeline->end) || - backup->stop_lsn <= timeline->end)) + lsn <= timeline->end)) return true; } return false; @@ -1684,7 +1971,7 @@ pgRecoveryTarget * parseRecoveryTargetOptions(const char *target_time, const char *target_xid, const char *target_inclusive, - TimeLineID target_tli, + const char *target_tli_string, const char *target_lsn, const char *target_stop, const char *target_name, @@ -1757,7 +2044,20 @@ parseRecoveryTargetOptions(const char *target_time, target_inclusive); } - rt->target_tli = target_tli; + rt->target_tli_string = target_tli_string; + rt->target_tli = 0; + /* target_tli can contains timeline number, "current" or "latest" */ + if(target_tli_string && strcmp(target_tli_string, "current") != 0 && strcmp(target_tli_string, "latest") != 0) + { + errno = 0; + rt->target_tli = strtoul(target_tli_string, NULL, 10); + if (errno == EINVAL || errno == ERANGE || !rt->target_tli) + { + elog(ERROR, "Invalid value for '--recovery-target-timeline' option '%s'", + target_tli_string); + } + } + if (target_stop) { if ((strcmp(target_stop, "immediate") != 0) @@ -1849,7 +2149,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, if (!database_map_file) elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", - base36enc(backup->start_time)); + backup_id_of(backup)); join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); @@ -1867,7 +2167,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* partial restore requested but database_map is missing */ if (!database_map) elog(ERROR, "Backup %s has empty or mangled database_map, partial restore is impossible.", - base36enc(backup->start_time)); + backup_id_of(backup)); /* * So we have a list of datnames and a database_map for it. @@ -1897,7 +2197,7 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* If specified datname is not found in database_map, error out */ if (!found_match) elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, base36enc(backup->start_time)); + datname, backup_id_of(backup)); } /* At this moment only databases to exclude are left in the map */ @@ -1935,14 +2235,14 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* If specified datname is not found in database_map, error out */ if (!found_match) elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, base36enc(backup->start_time)); + datname, backup_id_of(backup)); } } /* extra sanity: ensure that list is not empty */ if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", - base36enc(backup->start_time)); + backup_id_of(backup)); /* clean backup filelist */ if (files) @@ -1959,36 +2259,23 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, /* Check that instance is suitable for incremental restore * Depending on type of incremental restore requirements are differs. + * + * TODO: add PG_CONTROL_IS_MISSING */ -void +DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode) + IncrRestoreMode incremental_mode, + parray *partial_db_list, + bool allow_partial_incremental) { uint64 system_id_pgdata; + bool system_id_match = false; bool success = true; + bool postmaster_is_up = false; + bool backup_label_exists = false; pid_t pid; char backup_label[MAXPGPATH]; - /* slurp pg_control and check that system ID is the same */ - /* check that instance is not running */ - /* if lsn_based, check that there is no backup_label files is around AND - * get redo point lsn from destination pg_control. - - * It is really important to be sure that pg_control is in cohesion with - * data files content, because based on pg_control information we will - * choose a backup suitable for lsn based incremental restore. - */ - - system_id_pgdata = get_system_identifier(pgdata); - - if (system_id_pgdata != instance_config.system_identifier) - { - elog(WARNING, "Backup catalog was initialized for system id %lu, " - "but destination directory system id is %lu", - system_identifier, system_id_pgdata); - success = false; - } - /* check postmaster pid */ pid = fio_check_postmaster(pgdata, FIO_DB_HOST); @@ -1996,7 +2283,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, { char pid_file[MAXPGPATH]; - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); + join_path_components(pid_file, pgdata, "postmaster.pid"); elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", pid_file); success = false; @@ -2006,15 +2293,41 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", pid, pgdata); success = false; + postmaster_is_up = true; } + /* check that PG_VERSION is the same */ + + /* slurp pg_control and check that system ID is the same + * check that instance is not running + * if lsn_based, check that there is no backup_label files is around AND + * get redo point lsn from destination pg_control. + + * It is really important to be sure that pg_control is in cohesion with + * data files content, because based on pg_control information we will + * choose a backup suitable for lsn based incremental restore. + */ + elog(LOG, "Trying to read pg_control file in destination directory"); + + get_control_file_or_back_file(pgdata, FIO_DB_HOST, &instance_control); + control_downloaded = true; + + system_id_pgdata = instance_control.system_identifier; + + if (system_id_pgdata == instance_config.system_identifier) + system_id_match = true; + else + elog(WARNING, "Backup catalog was initialized for system id %lu, " + "but destination directory system id is %lu", + system_identifier, system_id_pgdata); + /* * TODO: maybe there should be some other signs, pointing to pg_control * desynchronization with cluster state. */ if (incremental_mode == INCR_LSN) { - snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata); + join_path_components(backup_label, pgdata, "backup_label"); if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " @@ -2023,9 +2336,28 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, "to cluster with pg_control not synchronized with cluster state." "Consider to use incremental restore in 'checksum' mode"); success = false; + backup_label_exists = true; } } + if (postmaster_is_up) + return POSTMASTER_IS_RUNNING; + + /* PG_CONTROL MISSING */ + + /* PG_CONTROL unreadable */ + + if (!system_id_match) + return SYSTEM_ID_MISMATCH; + + if (backup_label_exists) + return BACKUP_LABEL_EXISTS; + + if (partial_db_list && !allow_partial_incremental) + return PARTIAL_INCREMENTAL_FORBIDDEN; + /* some other error condition */ if (!success) - elog(ERROR, "Incremental restore is impossible"); + return DEST_IS_NOT_OK; + + return DEST_OK; } diff --git a/src/show.c b/src/show.c index 81a16ad64..810262df6 100644 --- a/src/show.c +++ b/src/show.c @@ -3,7 +3,7 @@ * show.c: show backup information. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -12,6 +12,7 @@ #include #include +#include #include #include "utils/json.h" @@ -54,32 +55,71 @@ typedef struct ShowArchiveRow static void show_instance_start(void); static void show_instance_end(void); -static void show_instance(const char *instance_name, time_t requested_backup_id, bool show_name); +static void show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name); static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup); -static int show_backup(const char *instance_name, time_t requested_backup_id); +static int show_backup(InstanceState *instanceState, time_t requested_backup_id); static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name); static void show_instance_json(const char *instance_name, parray *backup_list); -static void show_instance_archive(InstanceConfig *instance); +static void show_instance_archive(InstanceState *instanceState, InstanceConfig *instance); static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, parray *timelines_list, bool show_name); static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, parray *tli_list); +static bool backup_has_tablespace_map(pgBackup *backup); static PQExpBufferData show_buf; static bool first_instance = true; static int32 json_level = 0; +static const char* lc_env_locale; +typedef enum { + LOCALE_C, // Used for formatting output to unify the dot-based floating point representation + LOCALE_ENV // Default environment locale +} output_numeric_locale; + +#ifdef HAVE_USELOCALE +static locale_t env_locale, c_locale; +#endif +void memorize_environment_locale() { + lc_env_locale = (const char *)getenv("LC_NUMERIC"); + lc_env_locale = lc_env_locale != NULL ? lc_env_locale : "C"; +#ifdef HAVE_USELOCALE + env_locale = newlocale(LC_NUMERIC_MASK, lc_env_locale, (locale_t)0); + c_locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0); +#else +#ifdef HAVE__CONFIGTHREADLOCALE + _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); +#endif +#endif +} + +void free_environment_locale() { +#ifdef HAVE_USELOCALE + freelocale(env_locale); + freelocale(c_locale); +#endif +} + +static void set_output_numeric_locale(output_numeric_locale loc) { +#ifdef HAVE_USELOCALE + uselocale(loc == LOCALE_C ? c_locale : env_locale); +#else + setlocale(LC_NUMERIC, loc == LOCALE_C ? "C" : lc_env_locale); +#endif +} + /* * Entry point of pg_probackup SHOW subcommand. */ int -do_show(const char *instance_name, time_t requested_backup_id, bool show_archive) +do_show(CatalogState *catalogState, InstanceState *instanceState, + time_t requested_backup_id, bool show_archive) { int i; - if (instance_name == NULL && + if (instanceState == NULL && requested_backup_id != INVALID_BACKUP_ID) elog(ERROR, "You must specify --instance to use (-i, --backup-id) option"); @@ -88,25 +128,25 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive elog(ERROR, "You cannot specify --archive and (-i, --backup-id) options together"); /* - * if instance_name is not specified, + * if instance is not specified, * show information about all instances in this backup catalog */ - if (instance_name == NULL) + if (instanceState == NULL) { - parray *instances = catalog_get_instance_list(); + parray *instances = catalog_get_instance_list(catalogState); show_instance_start(); for (i = 0; i < parray_num(instances); i++) { - InstanceConfig *instance = parray_get(instances, i); - char backup_instance_path[MAXPGPATH]; + instanceState = parray_get(instances, i); - sprintf(backup_instance_path, "%s/%s/%s", backup_path, BACKUPS_DIR, instance->name); + if (interrupted) + elog(ERROR, "Interrupted during show"); if (show_archive) - show_instance_archive(instance); + show_instance_archive(instanceState, instanceState->config); else - show_instance(instance->name, INVALID_BACKUP_ID, true); + show_instance(instanceState, INVALID_BACKUP_ID, true); } show_instance_end(); @@ -120,11 +160,11 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive if (show_archive) { - InstanceConfig *instance = readInstanceConfigFile(instance_name); - show_instance_archive(instance); + InstanceConfig *instance = readInstanceConfigFile(instanceState); + show_instance_archive(instanceState, instance); } else - show_instance(instance_name, requested_backup_id, false); + show_instance(instanceState, requested_backup_id, false); show_instance_end(); @@ -134,11 +174,11 @@ do_show(const char *instance_name, time_t requested_backup_id, bool show_archive { if (show_archive) { - InstanceConfig *instance = readInstanceConfigFile(instance_name); - show_instance_archive(instance); + InstanceConfig *instance = readInstanceConfigFile(instanceState); + show_instance_archive(instanceState, instance); } else - show_backup(instance_name, requested_backup_id); + show_backup(instanceState, requested_backup_id); return 0; } @@ -163,22 +203,22 @@ pretty_size(int64 size, char *buf, size_t len) return; } - if (Abs(size) < limit) + if (size < limit) snprintf(buf, len, "%dB", (int) size); else { size >>= 9; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dkB", (int) half_rounded(size)); else { size >>= 10; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dMB", (int) half_rounded(size)); else { size >>= 10; - if (Abs(size) < limit2) + if (size < limit2) snprintf(buf, len, "%dGB", (int) half_rounded(size)); else { @@ -286,16 +326,16 @@ show_instance_end(void) * Show brief meta information about all backups in the backup instance. */ static void -show_instance(const char *instance_name, time_t requested_backup_id, bool show_name) +show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name) { parray *backup_list; - backup_list = catalog_get_backup_list(instance_name, requested_backup_id); + backup_list = catalog_get_backup_list(instanceState, requested_backup_id); if (show_format == SHOW_PLAIN) - show_instance_plain(instance_name, backup_list, show_name); + show_instance_plain(instanceState->instance_name, backup_list, show_name); else if (show_format == SHOW_JSON) - show_instance_json(instance_name, backup_list); + show_instance_json(instanceState->instance_name, backup_list); else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -314,14 +354,14 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) json_add(buf, JT_BEGIN_OBJECT, &json_level); - json_add_value(buf, "id", base36enc(backup->start_time), json_level, + json_add_value(buf, "id", backup_id_of(backup), json_level, true); if (backup->parent_backup != 0) json_add_value(buf, "parent-backup-id", base36enc(backup->parent_backup), json_level, true); - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup), + json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup, false), json_level, true); json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", @@ -371,12 +411,12 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); json_add_value(buf, "stop-lsn", lsn, json_level, true); - time2iso(timestamp, lengthof(timestamp), backup->start_time); + time2iso(timestamp, lengthof(timestamp), backup->start_time, false); json_add_value(buf, "start-time", timestamp, json_level, true); if (backup->end_time) { - time2iso(timestamp, lengthof(timestamp), backup->end_time); + time2iso(timestamp, lengthof(timestamp), backup->end_time, false); json_add_value(buf, "end-time", timestamp, json_level, true); } @@ -385,13 +425,13 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) if (backup->recovery_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time); + time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); json_add_value(buf, "recovery-time", timestamp, json_level, true); } if (backup->expire_time > 0) { - time2iso(timestamp, lengthof(timestamp), backup->expire_time); + time2iso(timestamp, lengthof(timestamp), backup->expire_time, false); json_add_value(buf, "expire-time", timestamp, json_level, true); } @@ -413,7 +453,7 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) appendPQExpBuffer(buf, INT64_FORMAT, backup->uncompressed_bytes); } - if (backup->uncompressed_bytes >= 0) + if (backup->pgdata_bytes >= 0) { json_add_key(buf, "pgdata-bytes", json_level); appendPQExpBuffer(buf, INT64_FORMAT, backup->pgdata_bytes); @@ -440,6 +480,32 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) appendPQExpBuffer(buf, "%u", backup->content_crc); } + /* print tablespaces list */ + if (backup_has_tablespace_map(backup)) + { + parray *links = parray_new(); + + json_add_key(buf, "tablespace_map", json_level); + json_add(buf, JT_BEGIN_ARRAY, &json_level); + + read_tablespace_map(links, backup->root_dir); + parray_qsort(links, pgFileCompareLinked); + + for (size_t i = 0; i < parray_num(links); i++){ + pgFile *link = (pgFile *) parray_get(links, i); + if (i) + appendPQExpBufferChar(buf, ','); + json_add(buf, JT_BEGIN_OBJECT, &json_level); + json_add_value(buf, "oid", link->name, json_level, true); + json_add_value(buf, "path", link->linked, json_level, true); + json_add(buf, JT_END_OBJECT, &json_level); + } + /* End of tablespaces */ + json_add(buf, JT_END_ARRAY, &json_level); + parray_walk(links, pgFileFree); + parray_free(links); + } + json_add(buf, JT_END_OBJECT, &json_level); } @@ -447,13 +513,14 @@ print_backup_json_object(PQExpBuffer buf, pgBackup *backup) * Show detailed meta information about specified backup. */ static int -show_backup(const char *instance_name, time_t requested_backup_id) +show_backup(InstanceState *instanceState, time_t requested_backup_id) { int i; pgBackup *backup = NULL; parray *backups; - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + //TODO pass requested_backup_id to the function + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Find requested backup */ for (i = 0; i < parray_num(backups); i++) @@ -474,12 +541,34 @@ show_backup(const char *instance_name, time_t requested_backup_id) elog(INFO, "Requested backup \"%s\" is not found.", /* We do not need free base36enc's result, we exit anyway */ base36enc(requested_backup_id)); + parray_walk(backups, pgBackupFree); + parray_free(backups); /* This is not error */ return 0; } if (show_format == SHOW_PLAIN) - pgBackupWriteControl(stdout, backup); + { + pgBackupWriteControl(stdout, backup, false); + + /* print tablespaces list */ + if (backup_has_tablespace_map(backup)) + { + parray *links = parray_new(); + + fio_fprintf(stdout, "\ntablespace_map = '"); + + read_tablespace_map(links, backup->root_dir); + parray_qsort(links, pgFileCompareLinked); + + for (size_t i = 0; i < parray_num(links); i++){ + pgFile *link = (pgFile *) parray_get(links, i); + fio_fprintf(stdout, "%s %s%s", link->name, link->linked, (i < parray_num(links) - 1) ? "; " : "'\n"); + } + parray_walk(links, pgFileFree); + parray_free(links); + } + } else elog(ERROR, "Invalid show format %d", (int) show_format); @@ -511,6 +600,9 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na ShowBackendRow *rows; TimeLineID parent_tli = 0; + // Since we've been printing a table, set LC_NUMERIC to its default environment value + set_output_numeric_locale(LOCALE_ENV); + for (i = 0; i < SHOW_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -540,22 +632,22 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na /* ID */ snprintf(row->backup_id, lengthof(row->backup_id), "%s", - base36enc(backup->start_time)); + backup_id_of(backup)); widths[cur] = Max(widths[cur], strlen(row->backup_id)); cur++; /* Recovery Time */ if (backup->recovery_time != (time_t) 0) time2iso(row->recovery_time, lengthof(row->recovery_time), - backup->recovery_time); + backup->recovery_time, false); else - StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time)); + strlcpy(row->recovery_time, "----", sizeof(row->recovery_time)); widths[cur] = Max(widths[cur], strlen(row->recovery_time)); cur++; /* Mode */ - row->mode = pgBackupGetBackupMode(backup); - widths[cur] = Max(widths[cur], strlen(row->mode)); + row->mode = pgBackupGetBackupMode(backup, show_color); + widths[cur] = Max(widths[cur], strlen(row->mode) - (show_color ? TC_LEN : 0)); cur++; /* WAL mode*/ @@ -584,7 +676,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na pretty_time_interval(difftime(backup->end_time, backup->start_time), row->duration, lengthof(row->duration)); else - StrNCpy(row->duration, "----", sizeof(row->duration)); + strlcpy(row->duration, "----", sizeof(row->duration)); widths[cur] = Max(widths[cur], strlen(row->duration)); cur++; @@ -628,8 +720,9 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na cur++; /* Status */ - row->status = status2str(backup->status); - widths[cur] = Max(widths[cur], strlen(row->status)); + row->status = show_color ? status2str_color(backup->status) : status2str(backup->status); + widths[cur] = Max(widths[cur], strlen(row->status) - (show_color ? TC_LEN : 0)); + } for (i = 0; i < SHOW_FIELDS_COUNT; i++) @@ -679,7 +772,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->recovery_time); cur++; - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), row->mode); cur++; @@ -715,7 +808,7 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na row->stop_lsn); cur++; - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], + appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), row->status); cur++; @@ -723,6 +816,8 @@ show_instance_plain(const char *instance_name, parray *backup_list, bool show_na } pfree(rows); + // Restore the C locale + set_output_numeric_locale(LOCALE_C); } /* @@ -771,16 +866,16 @@ show_instance_json(const char *instance_name, parray *backup_list) * show information about WAL archive of the instance */ static void -show_instance_archive(InstanceConfig *instance) +show_instance_archive(InstanceState *instanceState, InstanceConfig *instance) { parray *timelineinfos; - timelineinfos = catalog_get_timelines(instance); + timelineinfos = catalog_get_timelines(instanceState, instance); if (show_format == SHOW_PLAIN) - show_archive_plain(instance->name, instance->xlog_seg_size, timelineinfos, true); + show_archive_plain(instanceState->instance_name, instance->xlog_seg_size, timelineinfos, true); else if (show_format == SHOW_JSON) - show_archive_json(instance->name, instance->xlog_seg_size, timelineinfos); + show_archive_json(instanceState->instance_name, instance->xlog_seg_size, timelineinfos); else elog(ERROR, "Invalid show format %d", (int) show_format); } @@ -803,6 +898,9 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, uint32 widths_sum = 0; ShowArchiveRow *rows; + // Since we've been printing a table, set LC_NUMERIC to its default environment value + set_output_numeric_locale(LOCALE_ENV); + for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) widths[i] = strlen(names[i]); @@ -970,6 +1068,8 @@ show_archive_plain(const char *instance_name, uint32 xlog_seg_size, } pfree(rows); + // Restore the C locale + set_output_numeric_locale(LOCALE_C); //TODO: free timelines } @@ -1042,13 +1142,14 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, appendPQExpBuffer(buf, "%lu", tlinfo->size); json_add_key(buf, "zratio", json_level); + if (tlinfo->size != 0) - zratio = ((float)xlog_seg_size*tlinfo->n_xlog_files) / tlinfo->size; + zratio = ((float) xlog_seg_size * tlinfo->n_xlog_files) / tlinfo->size; appendPQExpBuffer(buf, "%.2f", zratio); if (tlinfo->closest_backup != NULL) snprintf(tmp_buf, lengthof(tmp_buf), "%s", - base36enc(tlinfo->closest_backup->start_time)); + backup_id_of(tlinfo->closest_backup)); else snprintf(tmp_buf, lengthof(tmp_buf), "%s", ""); @@ -1120,3 +1221,10 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size, first_instance = false; } + +static bool backup_has_tablespace_map(pgBackup *backup) +{ + char map_path[MAXPGPATH]; + join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); + return fileExists(map_path, FIO_BACKUP_HOST); +} diff --git a/src/stream.c b/src/stream.c new file mode 100644 index 000000000..77453e997 --- /dev/null +++ b/src/stream.c @@ -0,0 +1,779 @@ +/*------------------------------------------------------------------------- + * + * stream.c: pg_probackup specific code for WAL streaming + * + * Portions Copyright (c) 2015-2020, Postgres Professional + * + *------------------------------------------------------------------------- + */ + +#include "pg_probackup.h" +#include "receivelog.h" +#include "streamutil.h" +#include "access/timeline.h" + +#include +#include + +/* + * global variable needed by ReceiveXlogStream() + * + * standby_message_timeout controls how often we send a message + * back to the primary letting it know our progress, in milliseconds. + * + * in pg_probackup we use a default setting = 10 sec + */ +static int standby_message_timeout = 10 * 1000; + +/* stop_backup_lsn is set by pg_stop_backup() to stop streaming */ +XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; +static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; + +/* + * How long we should wait for streaming end in seconds. + * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 + */ +static uint32 stream_stop_timeout = 0; +/* Time in which we started to wait for streaming end */ +static time_t stream_stop_begin = 0; + +/* + * We need to wait end of WAL streaming before execute pg_stop_backup(). + */ +typedef struct +{ + char basedir[MAXPGPATH]; + PGconn *conn; + + /* + * Return value from the thread. + * 0 means there is no error, 1 - there is an error. + */ + int ret; + + XLogRecPtr startpos; + TimeLineID starttli; +} StreamThreadArg; + +static pthread_t stream_thread; +static StreamThreadArg stream_thread_arg = {"", NULL, 1}; + +static parray *xlog_files_list = NULL; +static bool do_crc = true; + +static void IdentifySystem(StreamThreadArg *stream_thread_arg); +static int checkpoint_timeout(PGconn *backup_conn); +static void *StreamLog(void *arg); +static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, + bool segment_finished); +static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, + XLogRecPtr xlogpos, char *basedir, + uint32 xlog_seg_size); +static void add_history_file_to_filelist(parray *filelist, uint32 timeline, + char *basedir); + +/* + * Run IDENTIFY_SYSTEM through a given connection and + * check system identifier and timeline are matching + */ +static void +IdentifySystem(StreamThreadArg *stream_thread_arg) +{ + PGresult *res; + + uint64 stream_conn_sysidentifier = 0; + char *stream_conn_sysidentifier_str; + TimeLineID stream_conn_tli = 0; + + if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) + { + PQfinish(stream_thread_arg->conn); + /* + * Error message already written in CheckServerVersionForStreaming(). + * There's no hope of recovering from a version mismatch, so don't + * retry. + */ + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + /* + * Identify server, obtain server system identifier and timeline + */ + res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING,"Could not send replication command \"%s\": %s", + "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); + PQfinish(stream_thread_arg->conn); + elog(ERROR, "Cannot continue backup because stream connect has failed."); + } + + stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); + stream_conn_tli = atoll(PQgetvalue(res, 0, 1)); + + /* Additional sanity, primary for PG 9.5, + * where system id can be obtained only via "IDENTIFY SYSTEM" + */ + if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) + elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); + + if (stream_conn_sysidentifier != instance_config.system_identifier) + elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " + "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", + stream_conn_sysidentifier, instance_config.system_identifier); + + if (stream_conn_tli != current.tli) + elog(ERROR, "Timeline identifier mismatch. " + "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", + stream_conn_tli, current.tli); + + PQclear(res); +} + +/* + * Retrieve checkpoint_timeout GUC value in seconds. + */ +static int +checkpoint_timeout(PGconn *backup_conn) +{ + PGresult *res; + const char *val; + const char *hintmsg; + int val_int; + + res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); + val = PQgetvalue(res, 0, 0); + + if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) + { + PQclear(res); + if (hintmsg) + elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, + hintmsg); + else + elog(ERROR, "Invalid value of checkout_timeout %s", val); + } + + PQclear(res); + + return val_int; +} + +/* + * CreateReplicationSlot_compat() -- wrapper for CreateReplicationSlot() used in StreamLog() + * src/bin/pg_basebackup/streamutil.c + * CreateReplicationSlot() has different signatures on different PG versions: + * PG 15 + * bool + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_temporary, bool is_physical, bool reserve_wal, + * bool slot_exists_ok, bool two_phase) + * PG 11-14 + * bool + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_temporary, bool is_physical, bool reserve_wal, + * bool slot_exists_ok) + * PG 9.5-10 + * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, + * bool is_physical, bool slot_exists_ok) + * NOTE: PG 9.6 and 10 support reserve_wal in + * pg_catalog.pg_create_physical_replication_slot(slot_name name [, immediately_reserve boolean]) + * and + * CREATE_REPLICATION_SLOT slot_name { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin } + * replication protocol command, but CreateReplicationSlot() C function doesn't + */ +static bool +CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *plugin, + bool is_temporary, bool is_physical, + bool slot_exists_ok) +{ +#if PG_VERSION_NUM >= 150000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); +#elif PG_VERSION_NUM >= 110000 + return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, + /* reserve_wal = */ true, slot_exists_ok); +#elif PG_VERSION_NUM >= 100000 + /* + * PG-10 doesn't support creating temp_slot by calling CreateReplicationSlot(), but + * it will be created by setting StreamCtl.temp_slot later in StreamLog() + */ + if (!is_temporary) + return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); + else + return true; +#else + /* these parameters not supported in PG < 10 */ + Assert(!is_temporary); + return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); +#endif +} + +/* + * Start the log streaming + */ +static void * +StreamLog(void *arg) +{ + StreamThreadArg *stream_arg = (StreamThreadArg *) arg; + + /* + * Always start streaming at the beginning of a segment + */ + stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; + + xlog_files_list = parray_new(); + + /* Initialize timeout */ + stream_stop_begin = 0; + + /* Create repslot */ +#if PG_VERSION_NUM >= 100000 + if (temp_slot || perm_slot) + if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, temp_slot, true, false)) +#else + if (perm_slot) + if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, false, true, false)) +#endif + { + interrupted = true; + elog(ERROR, "Couldn't create physical replication slot %s", replication_slot); + } + + /* + * Start the replication + */ + if (replication_slot) + elog(LOG, "started streaming WAL at %X/%X (timeline %u) using%s slot %s", + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli, +#if PG_VERSION_NUM >= 100000 + temp_slot ? " temporary" : "", +#else + "", +#endif + replication_slot); + else + elog(LOG, "started streaming WAL at %X/%X (timeline %u)", + (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, + stream_arg->starttli); + +#if PG_VERSION_NUM >= 90600 + { + StreamCtl ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + + ctl.startpos = stream_arg->startpos; + ctl.timeline = stream_arg->starttli; + ctl.sysidentifier = NULL; + ctl.stream_stop = stop_streaming; + ctl.standby_message_timeout = standby_message_timeout; + ctl.partial_suffix = NULL; + ctl.synchronous = false; + ctl.mark_done = false; + +#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 150000 + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, + PG_COMPRESSION_NONE, + 0, + false); +#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */ + ctl.walmethod = CreateWalDirectoryMethod( + stream_arg->basedir, +// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, + 0, + false); +#endif /* PG_VERSION_NUM >= 150000 */ + ctl.replication_slot = replication_slot; + ctl.stop_socket = PGINVALID_SOCKET; + ctl.do_sync = false; /* We sync all files at the end of backup */ +// ctl.mark_done /* for future use in s3 */ +#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 + /* StreamCtl.temp_slot used only for PG-10, in PG>10, temp_slots are created by calling CreateReplicationSlot() */ + ctl.temp_slot = temp_slot; +#endif /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 */ +#else /* PG_VERSION_NUM < 100000 */ + ctl.basedir = (char *) stream_arg->basedir; +#endif /* PG_VERSION_NUM >= 100000 */ + + if (ReceiveXlogStream(stream_arg->conn, &ctl) == false) + { + interrupted = true; + elog(ERROR, "Problem in receivexlog"); + } + +#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 160000 + if (!ctl.walmethod->ops->finish(ctl.walmethod)) +#else + if (!ctl.walmethod->finish()) +#endif + { + interrupted = true; + elog(ERROR, "Could not finish writing WAL files: %s", + strerror(errno)); + } +#endif /* PG_VERSION_NUM >= 100000 */ + } +#else /* PG_VERSION_NUM < 90600 */ + /* PG-9.5 */ + if (ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, + NULL, (char *) stream_arg->basedir, stop_streaming, + standby_message_timeout, NULL, false, false) == false) + { + interrupted = true; + elog(ERROR, "Problem in receivexlog"); + } +#endif /* PG_VERSION_NUM >= 90600 */ + + /* be paranoid and sort xlog_files_list, + * so if stop_lsn segno is already in the list, + * then list must be sorted to detect duplicates. + */ + parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); + + /* Add the last segment to the list */ + add_walsegment_to_filelist(xlog_files_list, stream_arg->starttli, + stop_stream_lsn, (char *) stream_arg->basedir, + instance_config.xlog_seg_size); + + /* append history file to walsegment filelist */ + add_history_file_to_filelist(xlog_files_list, stream_arg->starttli, (char *) stream_arg->basedir); + + /* + * TODO: remove redundant WAL segments + * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 + */ + + elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", + (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); + stream_arg->ret = 0; + + PQfinish(stream_arg->conn); + stream_arg->conn = NULL; + + return NULL; +} + +/* + * for ReceiveXlogStream + * + * The stream_stop callback will be called every time data + * is received, and whenever a segment is completed. If it returns + * true, the streaming will stop and the function + * return. As long as it returns false, streaming will continue + * indefinitely. + * + * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is + * set by pg_stop_backup(). + * + */ +static bool +stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) +{ + static uint32 prevtimeline = 0; + static XLogRecPtr prevpos = InvalidXLogRecPtr; + + /* check for interrupt */ + if (interrupted || thread_interrupted) + elog(ERROR, "Interrupted during WAL streaming"); + + /* we assume that we get called once at the end of each segment */ + if (segment_finished) + { + elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), + (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + + add_walsegment_to_filelist(xlog_files_list, timeline, xlogpos, + (char*) stream_thread_arg.basedir, + instance_config.xlog_seg_size); + } + + /* + * Note that we report the previous, not current, position here. After a + * timeline switch, xlogpos points to the beginning of the segment because + * that's where we always begin streaming. Reporting the end of previous + * timeline isn't totally accurate, because the next timeline can begin + * slightly before the end of the WAL that we received on the previous + * timeline, but it's close enough for reporting purposes. + */ + if (prevtimeline != 0 && prevtimeline != timeline) + elog(LOG, _("switched to timeline %u at %X/%X\n"), + timeline, (uint32) (prevpos >> 32), (uint32) prevpos); + + if (!XLogRecPtrIsInvalid(stop_backup_lsn)) + { + if (xlogpos >= stop_backup_lsn) + { + stop_stream_lsn = xlogpos; + return true; + } + + /* pg_stop_backup() was executed, wait for the completion of stream */ + if (stream_stop_begin == 0) + { + elog(INFO, "Wait for LSN %X/%X to be streamed", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); + + stream_stop_begin = time(NULL); + } + + if (time(NULL) - stream_stop_begin > stream_stop_timeout) + elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", + (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, + stream_stop_timeout); + } + + prevtimeline = timeline; + prevpos = xlogpos; + + return false; +} + + +/* --- External API --- */ + +/* + * Maybe add a StreamOptions struct ? + * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. + */ +parray* +get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list) +{ + PGresult *res; + PGconn *conn; + char *history; + char query[128]; + parray *result = NULL; + parray *tli_list = NULL; + timelineInfo *tlinfo = NULL; + int i,j; + + snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); + + /* + * Connect in replication mode to the server. + */ + conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser, + false); + + if (!conn) + return NULL; + + res = PQexec(conn, query); + PQfinish(conn); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(WARNING, "Could not send replication command \"%s\": %s", + query, PQresultErrorMessage(res)); + PQclear(res); + return NULL; + } + + /* + * The response to TIMELINE_HISTORY is a single row result set + * with two fields: filename and content + */ + + if (PQnfields(res) != 2 || PQntuples(res) != 1) + { + elog(WARNING, "Unexpected response to TIMELINE_HISTORY command: " + "got %d rows and %d fields, expected %d rows and %d fields", + PQntuples(res), PQnfields(res), 1, 2); + PQclear(res); + return NULL; + } + + history = pgut_strdup(PQgetvalue(res, 0, 1)); + result = parse_tli_history_buffer(history, tli); + + /* some cleanup */ + pg_free(history); + PQclear(res); + + if (result) + tlinfo = timelineInfoNew(tli); + else + return NULL; + + /* transform TimeLineHistoryEntry into timelineInfo */ + for (i = parray_num(result) -1; i >= 0; i--) + { + TimeLineHistoryEntry *tln = (TimeLineHistoryEntry *) parray_get(result, i); + + tlinfo->parent_tli = tln->tli; + tlinfo->switchpoint = tln->end; + + if (!tli_list) + tli_list = parray_new(); + + parray_append(tli_list, tlinfo); + + /* Next tli */ + tlinfo = timelineInfoNew(tln->tli); + + /* oldest tli */ + if (i == 0) + { + tlinfo->tli = tln->tli; + tlinfo->parent_tli = 0; + tlinfo->switchpoint = 0; + parray_append(tli_list, tlinfo); + } + } + + /* link parent to child */ + for (i = 0; i < parray_num(tli_list); i++) + { + tlinfo = (timelineInfo *) parray_get(tli_list, i); + + for (j = 0; j < parray_num(tli_list); j++) + { + timelineInfo *tlinfo_parent = (timelineInfo *) parray_get(tli_list, j); + + if (tlinfo->parent_tli == tlinfo_parent->tli) + { + tlinfo->parent_link = tlinfo_parent; + break; + } + } + } + + /* add backups to each timeline info */ + for (i = 0; i < parray_num(tli_list); i++) + { + tlinfo = parray_get(tli_list, i); + for (j = 0; j < parray_num(backup_list); j++) + { + pgBackup *backup = parray_get(backup_list, j); + if (tlinfo->tli == backup->tli) + { + if (tlinfo->backups == NULL) + tlinfo->backups = parray_new(); + parray_append(tlinfo->backups, backup); + } + } + } + + /* cleanup */ + parray_walk(result, pg_free); + pg_free(result); + + return tli_list; +} + +parray* +parse_tli_history_buffer(char *history, TimeLineID tli) +{ + char *curLine = history; + TimeLineHistoryEntry *entry; + TimeLineHistoryEntry *last_timeline = NULL; + parray *result = NULL; + + /* Parse timeline history buffer string by string */ + while (curLine) + { + char tempStr[1024]; + char *nextLine = strchr(curLine, '\n'); + int curLineLen = nextLine ? (nextLine-curLine) : strlen(curLine); + + memcpy(tempStr, curLine, curLineLen); + tempStr[curLineLen] = '\0'; // NUL-terminate! + curLine = nextLine ? (nextLine+1) : NULL; + + if (curLineLen > 0) + { + char *ptr; + TimeLineID tli; + uint32 switchpoint_hi; + uint32 switchpoint_lo; + int nfields; + + for (ptr = tempStr; *ptr; ptr++) + { + if (!isspace((unsigned char) *ptr)) + break; + } + if (*ptr == '\0' || *ptr == '#') + continue; + + nfields = sscanf(tempStr, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); + + if (nfields < 1) + { + /* expect a numeric timeline ID as first field of line */ + elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a numeric timeline ID.", tempStr); + } + if (nfields != 3) + elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a transaction log switchpoint location.", tempStr); + + if (last_timeline && tli <= last_timeline->tli) + elog(ERROR, "Timeline IDs must be in increasing sequence: \"%s\"", tempStr); + + entry = pgut_new(TimeLineHistoryEntry); + entry->tli = tli; + entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; + + last_timeline = entry; + /* Build list with newest item first */ + if (!result) + result = parray_new(); + parray_append(result, entry); + elog(VERBOSE, "parse_tli_history_buffer() found entry: tli = %X, end = %X/%X", + tli, switchpoint_hi, switchpoint_lo); + + /* we ignore the remainder of each line */ + } + } + + return result; +} + +/* + * Maybe add a StreamOptions struct ? + * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. + */ +void +start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, + XLogRecPtr startpos, TimeLineID starttli, bool is_backup) +{ + /* calculate crc only when running backup, catchup has no need for it */ + do_crc = is_backup; + /* How long we should wait for streaming end after pg_stop_backup */ + stream_stop_timeout = checkpoint_timeout(backup_conn); + //TODO Add a comment about this calculation + stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; + + strlcpy(stream_thread_arg.basedir, stream_dst_path, sizeof(stream_thread_arg.basedir)); + + /* + * Connect in replication mode to the server. + */ + stream_thread_arg.conn = pgut_connect_replication(conn_opt->pghost, + conn_opt->pgport, + conn_opt->pgdatabase, + conn_opt->pguser, + true); + /* sanity check*/ + IdentifySystem(&stream_thread_arg); + + /* Set error exit code as default */ + stream_thread_arg.ret = 1; + /* we must use startpos as start_lsn from start_backup */ + stream_thread_arg.startpos = startpos; + stream_thread_arg.starttli = starttli; + + thread_interrupted = false; + pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); +} + +/* + * Wait for the completion of stream + * append list of streamed xlog files + * into backup_files_list (if it is not NULL) + */ +int +wait_WAL_streaming_end(parray *backup_files_list) +{ + pthread_join(stream_thread, NULL); + + if(backup_files_list != NULL) + parray_concat(backup_files_list, xlog_files_list); + parray_free(xlog_files_list); + return stream_thread_arg.ret; +} + +/* Append streamed WAL segment to filelist */ +void +add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) +{ + XLogSegNo xlog_segno; + char wal_segment_name[MAXFNAMELEN]; + char wal_segment_relpath[MAXPGPATH]; + char wal_segment_fullpath[MAXPGPATH]; + pgFile *file = NULL; + pgFile **existing_file = NULL; + + GetXLogSegNo(xlogpos, xlog_segno, xlog_seg_size); + + /* + * When xlogpos points to the zero offset (0/3000000), + * it means that previous segment was just successfully streamed. + * When xlogpos points to the positive offset, + * then current segment is successfully streamed. + */ + if (WalSegmentOffset(xlogpos, xlog_seg_size) == 0) + xlog_segno--; + + GetXLogFileName(wal_segment_name, timeline, xlog_segno, xlog_seg_size); + + join_path_components(wal_segment_fullpath, basedir, wal_segment_name); + join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); + + file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); + + /* + * Check if file is already in the list + * stop_lsn segment can be added to this list twice, so + * try not to add duplicates + */ + + existing_file = (pgFile **) parray_bsearch(filelist, file, pgFileCompareRelPathWithExternal); + + if (existing_file) + { + if (do_crc) + (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + (*existing_file)->write_size = xlog_seg_size; + (*existing_file)->uncompressed_size = xlog_seg_size; + + return; + } + + if (do_crc) + file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + + /* Should we recheck it using stat? */ + file->write_size = xlog_seg_size; + file->uncompressed_size = xlog_seg_size; + + /* append file to filelist */ + parray_append(filelist, file); +} + +/* Append history file to filelist */ +void +add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) +{ + char filename[MAXFNAMELEN]; + char fullpath[MAXPGPATH]; + char relpath[MAXPGPATH]; + pgFile *file = NULL; + + /* Timeline 1 does not have a history file */ + if (timeline == 1) + return; + + snprintf(filename, lengthof(filename), "%08X.history", timeline); + join_path_components(fullpath, basedir, filename); + join_path_components(relpath, PG_XLOG_DIR, filename); + + file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); + + /* calculate crc */ + if (do_crc) + file->crc = pgFileGetCRC(fullpath, true, false); + file->write_size = file->size; + file->uncompressed_size = file->size; + + /* append file to filelist */ + parray_append(filelist, file); +} diff --git a/src/util.c b/src/util.c index 5ad751df2..3c0a33453 100644 --- a/src/util.c +++ b/src/util.c @@ -3,15 +3,13 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2021, Postgres Professional * *------------------------------------------------------------------------- */ #include "pg_probackup.h" -#include "catalog/pg_control.h" - #include #include @@ -34,38 +32,22 @@ static const char *statusName[] = }; const char * -base36enc(long unsigned int value) +base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]) { const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ - static char buffer[14]; - unsigned int offset = sizeof(buffer); + char buffer[base36bufsize]; + char *p; - buffer[--offset] = '\0'; + p = &buffer[sizeof(buffer)-1]; + *p = '\0'; do { - buffer[--offset] = base36[value % 36]; + *(--p) = base36[value % 36]; } while (value /= 36); - return &buffer[offset]; -} - -/* - * Same as base36enc(), but the result must be released by the user. - */ -char * -base36enc_dup(long unsigned int value) -{ - const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - /* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ - char buffer[14]; - unsigned int offset = sizeof(buffer); - - buffer[--offset] = '\0'; - do { - buffer[--offset] = base36[value % 36]; - } while (value /= 36); + /* I know, it doesn't look safe */ + strncpy(buf, p, base36bufsize); - return strdup(&buffer[offset]); + return buf; } long unsigned int @@ -92,7 +74,7 @@ checkControlFile(ControlFileData *ControlFile) if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) && ControlFile->pg_control_version / 65536 != 0) - elog(ERROR, "possible byte ordering mismatch\n" + elog(ERROR, "Possible byte ordering mismatch\n" "The byte ordering used to store the pg_control file might not match the one\n" "used by this program. In that case the results below would be incorrect, and\n" "the PostgreSQL installation would be incompatible with this data directory."); @@ -111,7 +93,7 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) #endif if (size != ControlFileSize) - elog(ERROR, "unexpected control file size %d, expected %d", + elog(ERROR, "Unexpected control file size %d, expected %d", (int) size, ControlFileSize); memcpy(ControlFile, src, sizeof(ControlFileData)); @@ -136,7 +118,7 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo #endif /* copy controlFileSize */ - buffer = pg_malloc(ControlFileSize); + buffer = pg_malloc0(ControlFileSize); memcpy(buffer, ControlFile, sizeof(ControlFileData)); /* Write pg_control */ @@ -169,12 +151,12 @@ get_current_timeline(PGconn *conn) char *val; res = pgut_execute_extended(conn, - "SELECT timeline_id FROM pg_control_checkpoint()", 0, NULL, true, true); + "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()", 0, NULL, true, true); if (PQresultStatus(res) == PGRES_TUPLES_OK) val = PQgetvalue(res, 0, 0); else - return get_current_timeline_from_control(false); + return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); if (!parse_uint32(val, &tli, 0)) { @@ -182,7 +164,7 @@ get_current_timeline(PGconn *conn) elog(WARNING, "Invalid value of timeline_id %s", val); /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(false); + return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); } return tli; @@ -190,15 +172,15 @@ get_current_timeline(PGconn *conn) /* Get timeline from pg_control file */ TimeLineID -get_current_timeline_from_control(bool safe) +get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, - safe, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, + safe, location); if (safe && buffer == NULL) return 0; @@ -208,6 +190,26 @@ get_current_timeline_from_control(bool safe) return ControlFile.checkPointCopy.ThisTimeLineID; } +void +get_control_file_or_back_file(const char *pgdata_path, fio_location location, ControlFileData *control) +{ + char *buffer; + size_t size; + + /* First fetch file... */ + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, true, location); + + if (!buffer || size == 0){ + /* Error read XLOG_CONTROL_FILE or file is truncated, trying read backup */ + buffer = slurpFile(pgdata_path, XLOG_CONTROL_BAK_FILE, &size, true, location); + if (!buffer) + elog(ERROR, "Could not read %s and %s files\n", XLOG_CONTROL_FILE, XLOG_CONTROL_BAK_FILE); /* Maybe it should be PANIC? */ + } + digestControlFile(control, buffer, size); + pg_free(buffer); +} + + /* * Get last check point record ptr from pg_tonrol. */ @@ -249,15 +251,15 @@ get_checkpoint_location(PGconn *conn) } uint64 -get_system_identifier(const char *pgdata_path) +get_system_identifier(const char *pgdata_path, fio_location location, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - if (buffer == NULL) + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location); + if (safe && buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -299,7 +301,7 @@ get_remote_system_identifier(PGconn *conn) } uint32 -get_xlog_seg_size(char *pgdata_path) +get_xlog_seg_size(const char *pgdata_path) { #if PG_VERSION_NUM >= 110000 ControlFileData ControlFile; @@ -352,14 +354,14 @@ get_pgcontrol_checksum(const char *pgdata_path) } void -get_redo(const char *pgdata_path, RedoParams *redo) +get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -418,7 +420,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, FIN_CRC32C(ControlFile.crc); /* overwrite pg_control */ - snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE); + join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE); writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST); /* Update pg_control checksum in backup_list */ @@ -516,6 +518,29 @@ status2str(BackupStatus status) return statusName[status]; } +const char * +status2str_color(BackupStatus status) +{ + char *status_str = pgut_malloc(20); + + /* UNKNOWN */ + if (status == BACKUP_STATUS_INVALID) + snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, "UNKNOWN", TC_RESET); + /* CORRUPT, ERROR and ORPHAN */ + else if (status == BACKUP_STATUS_CORRUPT || status == BACKUP_STATUS_ERROR || + status == BACKUP_STATUS_ORPHAN) + snprintf(status_str, 20, "%s%s%s", TC_RED_BOLD, statusName[status], TC_RESET); + /* MERGING, MERGED, DELETING and DELETED */ + else if (status == BACKUP_STATUS_MERGING || status == BACKUP_STATUS_MERGED || + status == BACKUP_STATUS_DELETING || status == BACKUP_STATUS_DELETED) + snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, statusName[status], TC_RESET); + /* OK and DONE */ + else + snprintf(status_str, 20, "%s%s%s", TC_GREEN_BOLD, statusName[status], TC_RESET); + + return status_str; +} + BackupStatus str2status(const char *status) { @@ -538,33 +563,7 @@ datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) offset = blkno / 8; bitno = blkno % 8; - /* enlarge or create bitmap if needed */ - if (map->bitmapsize <= offset) - { - int oldsize = map->bitmapsize; - int newsize; - - /* - * The minimum to hold the new bit is offset + 1. But add some - * headroom, so that we don't need to repeatedly enlarge the bitmap in - * the common case that blocks are modified in order, from beginning - * of a relation to the end. - */ - newsize = offset + 1; - newsize += 10; - - map->bitmap = pg_realloc(map->bitmap, newsize); - - /* zero out the newly allocated region */ - memset(&map->bitmap[oldsize], 0, newsize - oldsize); - - map->bitmapsize = newsize; - } - - //datapagemap_print(map); - - /* check the bit */ - return map->bitmap[offset] & (1 << bitno); + return (map->bitmapsize <= offset) ? false : (map->bitmap[offset] & (1 << bitno)) != 0; } /* @@ -578,55 +577,27 @@ datapagemap_print_debug(datapagemap_t *map) iter = datapagemap_iterate(map); while (datapagemap_next(iter, &blocknum)) - elog(INFO, " block %u", blocknum); + elog(VERBOSE, " block %u", blocknum); pg_free(iter); } -/* - * Return pid of postmaster process running in given pgdata. - * Return 0 if there is none. - * Return 1 if postmaster.pid is mangled. - */ -pid_t -check_postmaster(const char *pgdata) +const char* +backup_id_of(pgBackup *backup) { - FILE *fp; - pid_t pid; - char pid_file[MAXPGPATH]; - - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata); - - fp = fopen(pid_file, "r"); - if (fp == NULL) - { - /* No pid file, acceptable*/ - if (errno == ENOENT) - return 0; - else - elog(ERROR, "Cannot open file \"%s\": %s", - pid_file, strerror(errno)); - } + /* Change this Assert when backup_id will not be bound to start_time */ + Assert(backup->backup_id == backup->start_time || backup->start_time == 0); - if (fscanf(fp, "%i", &pid) != 1) + if (backup->backup_id_encoded[0] == '\x00') { - /* something is wrong with the file content */ - pid = 1; - } - - if (pid > 1) - { - if (kill(pid, 0) != 0) - { - /* process no longer exists */ - if (errno == ESRCH) - pid = 0; - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - pid, strerror(errno)); - } + base36enc_to(backup->backup_id, backup->backup_id_encoded); } + return backup->backup_id_encoded; +} - fclose(fp); - return pid; +void +reset_backup_id(pgBackup *backup) +{ + backup->backup_id = INVALID_BACKUP_ID; + memset(backup->backup_id_encoded, 0, sizeof(backup->backup_id_encoded)); } diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 1ef332ed5..f049aa1be 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -18,7 +18,11 @@ #include "getopt_long.h" +#ifndef WIN32 +#include +#endif #include +#include #define MAXPG_LSNCOMPONENT 8 @@ -87,6 +91,63 @@ static const unit_conversion time_unit_conversion_table[] = {""} /* end of table marker */ }; +/* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ +static char const * const subcmd_names[] = +{ + "NO_CMD", + "init", + "add-instance", + "del-instance", + "archive-push", + "archive-get", + "backup", + "restore", + "validate", + "delete", + "merge", + "show", + "set-config", + "set-backup", + "show-config", + "checkdb", + "ssh", + "agent", + "help", + "version", + "catchup", +}; + +ProbackupSubcmd +parse_subcmd(char const * const subcmd_str) +{ + struct { + ProbackupSubcmd id; + char *name; + } + static const subcmd_additional_names[] = { + { HELP_CMD, "--help" }, + { HELP_CMD, "-?" }, + { VERSION_CMD, "--version" }, + { VERSION_CMD, "-V" }, + }; + + int i; + for(i = (int)NO_CMD + 1; i < sizeof(subcmd_names) / sizeof(subcmd_names[0]); ++i) + if(strcmp(subcmd_str, subcmd_names[i]) == 0) + return (ProbackupSubcmd)i; + for(i = 0; i < sizeof(subcmd_additional_names) / sizeof(subcmd_additional_names[0]); ++i) + if(strcmp(subcmd_str, subcmd_additional_names[i].name) == 0) + return subcmd_additional_names[i].id; + return NO_CMD; +} + +char const * +get_subcmd_name(ProbackupSubcmd const subcmd) +{ + Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); + return subcmd_names[(int)subcmd]; +} + /* * Reading functions. */ @@ -460,17 +521,22 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], optstring = longopts_to_optstring(longopts, cmd_len + len); + opterr = 0; /* Assign named options */ while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) { ConfigOption *opt; + if (c == '?') + { + elog(ERROR, "Option '%s' requires an argument. Try \"%s --help\" for more information.", + argv[optind-1], PROGRAM_NAME); + } opt = option_find(c, cmd_options); if (opt == NULL) opt = option_find(c, options); if (opt - && !remote_agent && opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) elog(ERROR, "Option %s cannot be specified in command line", opt->lname); @@ -478,6 +544,9 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[], assign_option(opt, optarg, SOURCE_CMD); } + pgut_free(optstring); + pgut_free(longopts); + return optind; } @@ -490,9 +559,9 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, bool strict, bool missing_ok) { FILE *fp; - char buf[1024]; + char buf[4096]; char key[1024]; - char value[1024]; + char value[2048]; int parsed_options = 0; if (!options) @@ -609,6 +678,8 @@ config_set_opt(ConfigOption options[], void *var, OptionSource source) /* * Return value of the function in the string representation. Result is * allocated string. + * We can set GET_VAL_IN_BASE_UNITS flag in opt->flags + * before call option_get_value() to get option value in default units */ char * option_get_value(ConfigOption *opt) @@ -623,20 +694,33 @@ option_get_value(ConfigOption *opt) */ if (opt->flags & OPTION_UNIT) { - if (opt->type == 'i') - convert_from_base_unit(*((int32 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'i') - convert_from_base_unit(*((int64 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'u') - convert_from_base_unit_u(*((uint32 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); - else if (opt->type == 'U') - convert_from_base_unit_u(*((uint64 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); + if (opt->flags & GET_VAL_IN_BASE_UNITS){ + if (opt->type == 'i') + value = *((int32 *) opt->var); + else if (opt->type == 'I') + value = *((int64 *) opt->var); + else if (opt->type == 'u') + value_u = *((uint32 *) opt->var); + else if (opt->type == 'U') + value_u = *((uint64 *) opt->var); + unit = ""; + } + else + { + if (opt->type == 'i') + convert_from_base_unit(*((int32 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'I') + convert_from_base_unit(*((int64 *) opt->var), + opt->flags & OPTION_UNIT, &value, &unit); + else if (opt->type == 'u') + convert_from_base_unit_u(*((uint32 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + else if (opt->type == 'U') + convert_from_base_unit_u(*((uint64 *) opt->var), + opt->flags & OPTION_UNIT, &value_u, &unit); + } } - /* Get string representation itself */ switch (opt->type) { @@ -679,7 +763,7 @@ option_get_value(ConfigOption *opt) if (t > 0) { timestamp = palloc(100); - time2iso(timestamp, 100, t); + time2iso(timestamp, 100, t, false); } else timestamp = palloc0(1 /* just null termination */); @@ -1111,8 +1195,11 @@ parse_time(const char *value, time_t *result, bool utc_default) struct tm tm; char junk[2]; + char *local_tz = getenv("TZ"); + /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + + 1); + tmp = pgut_malloc(strlen(value) + 1); + if(!tmp) return false; len = 0; fields_num = 1; @@ -1140,7 +1227,10 @@ parse_time(const char *value, time_t *result, bool utc_default) errno = 0; hr = strtol(value + 1, &cp, 10); if ((value + 1) == cp || errno == ERANGE) + { + pfree(tmp); return false; + } /* explicit delimiter? */ if (*cp == ':') @@ -1148,13 +1238,19 @@ parse_time(const char *value, time_t *result, bool utc_default) errno = 0; min = strtol(cp + 1, &cp, 10); if (errno == ERANGE) + { + pfree(tmp); return false; + } if (*cp == ':') { errno = 0; sec = strtol(cp + 1, &cp, 10); if (errno == ERANGE) + { + pfree(tmp); return false; + } } } /* otherwise, might have run things together... */ @@ -1169,11 +1265,20 @@ parse_time(const char *value, time_t *result, bool utc_default) /* Range-check the values; see notes in datatype/timestamp.h */ if (hr < 0 || hr > MAX_TZDISP_HOUR) + { + pfree(tmp); return false; + } if (min < 0 || min >= MINS_PER_HOUR) + { + pfree(tmp); return false; + } if (sec < 0 || sec >= SECS_PER_MINUTE) + { + pfree(tmp); return false; + } tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; if (*value == '-') @@ -1186,7 +1291,10 @@ parse_time(const char *value, time_t *result, bool utc_default) } /* wrong format */ else if (!IsSpace(*value)) + { + pfree(tmp); return false; + } else value++; } @@ -1203,7 +1311,7 @@ parse_time(const char *value, time_t *result, bool utc_default) i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", &tm.tm_year, &tm.tm_mon, &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); - free(tmp); + pfree(tmp); if (i < 3 || i > 6) return false; @@ -1221,24 +1329,33 @@ parse_time(const char *value, time_t *result, bool utc_default) /* determine whether Daylight Saving Time is in effect */ tm.tm_isdst = -1; + /* + * If tz is not set, + * treat it as UTC if requested, otherwise as local timezone + */ + if (tz_set || utc_default) + { + /* set timezone to UTC */ + pgut_setenv("TZ", "UTC"); + tzset(); + } + + /* convert time to utc unix time */ *result = mktime(&tm); + /* return old timezone back if any */ + if (local_tz) + pgut_setenv("TZ", local_tz); + else + pgut_unsetenv("TZ"); + + tzset(); + /* adjust time zone */ if (tz_set || utc_default) { - time_t ltime = time(NULL); - struct tm *ptm = gmtime(<ime); - time_t gmt = mktime(ptm); - time_t offset; - /* UTC time */ *result -= tz; - - /* Get local time */ - ptm = localtime(<ime); - offset = ltime - gmt + (ptm->tm_isdst ? 3600 : 0); - - *result += offset; } return true; @@ -1343,16 +1460,16 @@ parse_lsn(const char *value, XLogRecPtr *result) len1 = strspn(value, "0123456789abcdefABCDEF"); if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; else { - elog(ERROR, "invalid LSN \"%s\"", value); + elog(ERROR, "Invalid LSN \"%s\"", value); return false; } @@ -1462,14 +1579,26 @@ convert_from_base_unit_u(uint64 base_value, int base_unit, * Convert time_t value to ISO-8601 format string. Always set timezone offset. */ void -time2iso(char *buf, size_t len, time_t time) +time2iso(char *buf, size_t len, time_t time, bool utc) { - struct tm *ptm = gmtime(&time); - time_t gmt = mktime(ptm); + struct tm *ptm = NULL; + time_t gmt; time_t offset; char *ptr = buf; + /* set timezone to UTC if requested */ + if (utc) + { + ptm = gmtime(&time); + strftime(ptr, len, "%Y-%m-%d %H:%M:%S+00", ptm); + return; + } + + ptm = gmtime(&time); + gmt = mktime(ptm); ptm = localtime(&time); + + /* adjust timezone offset */ offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 46b5d6c1b..59da29bd5 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -16,6 +16,32 @@ #define INFINITE_STR "INFINITE" +/* Order is important, keep it in sync with configuration.c:subcmd_names[] and help.c:help_command() */ +typedef enum ProbackupSubcmd +{ + NO_CMD = 0, + INIT_CMD, + ADD_INSTANCE_CMD, + DELETE_INSTANCE_CMD, + ARCHIVE_PUSH_CMD, + ARCHIVE_GET_CMD, + BACKUP_CMD, + RESTORE_CMD, + VALIDATE_CMD, + DELETE_CMD, + MERGE_CMD, + SHOW_CMD, + SET_CONFIG_CMD, + SET_BACKUP_CMD, + SHOW_CONFIG_CMD, + CHECKDB_CMD, + SSH_CMD, + AGENT_CMD, + HELP_CMD, + VERSION_CMD, + CATCHUP_CMD, +} ProbackupSubcmd; + typedef enum OptionSource { SOURCE_DEFAULT, @@ -35,14 +61,14 @@ typedef char *(*option_get_fn) (ConfigOption *opt); /* * type: - * b: bool (true) - * B: bool (false) + * b: bool (true) + * B: bool (false) * f: option_fn - * i: 32bit signed integer - * u: 32bit unsigned integer - * I: 64bit signed integer - * U: 64bit unsigned integer - * s: string + * i: 32bit signed integer + * u: 32bit unsigned integer + * I: 64bit signed integer + * U: 64bit unsigned integer + * s: string * t: time_t */ struct ConfigOption @@ -74,7 +100,10 @@ struct ConfigOption #define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) +#define GET_VAL_IN_BASE_UNITS 0x80000000 /* bitflag to get memory and time values in default units*/ +extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); +extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], ConfigOption options[]); extern int config_read_opt(const char *path, ConfigOption options[], int elevel, @@ -96,7 +125,7 @@ extern bool parse_int(const char *value, int *result, int flags, const char **hintmsg); extern bool parse_lsn(const char *value, XLogRecPtr *result); -extern void time2iso(char *buf, size_t len, time_t time); +extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern void convert_from_base_unit(int64 base_value, int base_unit, int64 *value, const char **unit); diff --git a/src/utils/file.c b/src/utils/file.c index db5924a7c..fa08939f5 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1,8 +1,10 @@ #include #include -#include #include "pg_probackup.h" +/* sys/stat.h must be included after pg_probackup.h (see problems with compilation for windows described in PGPRO-5750) */ +#include + #include "file.h" #include "storage/checksum.h" @@ -14,6 +16,11 @@ static __thread void* fio_stdin_buffer; static __thread int fio_stdout = 0; static __thread int fio_stdin = 0; static __thread int fio_stderr = 0; +static char *async_errormsg = NULL; + +#define PAGE_ZEROSEARCH_COARSE_GRANULARITY 4096 +#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64 +static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0}; fio_location MyLocation; @@ -29,7 +36,6 @@ typedef struct int path_len; } fio_send_request; - typedef struct { char path[MAXPGPATH]; @@ -48,7 +54,6 @@ typedef struct size_t size; time_t mtime; bool is_datafile; - bool is_database; Oid tblspcOid; Oid dbOid; Oid relOid; @@ -83,15 +88,35 @@ typedef struct #undef fopen(a, b) #endif +void +setMyLocation(ProbackupSubcmd const subcmd) +{ + +#ifdef WIN32 + if (IsSshProtocol()) + elog(ERROR, "Currently remote operations on Windows are not supported"); +#endif + + MyLocation = IsSshProtocol() + ? (subcmd == ARCHIVE_PUSH_CMD || subcmd == ARCHIVE_GET_CMD) + ? FIO_DB_HOST + : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD || subcmd == CATCHUP_CMD) + ? FIO_BACKUP_HOST + : FIO_LOCAL_HOST + : FIO_LOCAL_HOST; +} + /* Use specified file descriptors as stdin/stdout for FIO functions */ -void fio_redirect(int in, int out, int err) +void +fio_redirect(int in, int out, int err) { fio_stdin = in; fio_stdout = out; fio_stderr = err; } -void fio_error(int rc, int size, char const* file, int line) +void +fio_error(int rc, int size, char const* file, int line) { if (remote_agent) { @@ -114,7 +139,8 @@ void fio_error(int rc, int size, char const* file, int line) } /* Check if file descriptor is local or remote (created by FIO) */ -static bool fio_is_remote_fd(int fd) +static bool +fio_is_remote_fd(int fd) { return (fd & FIO_PIPE_MARKER) != 0; } @@ -156,14 +182,17 @@ fio_safestat(const char *path, struct stat *buf) #define stat(x, y) fio_safestat(x, y) /* TODO: use real pread on Linux */ -static ssize_t pread(int fd, void* buf, size_t size, off_t off) +static ssize_t +pread(int fd, void* buf, size_t size, off_t off) { off_t rc = lseek(fd, off, SEEK_SET); if (rc != off) return -1; return read(fd, buf, size); } -static int remove_file_or_dir(char const* path) + +static int +remove_file_or_dir(char const* path) { int rc = remove(path); #ifdef WIN32 @@ -177,7 +206,8 @@ static int remove_file_or_dir(char const* path) #endif /* Check if specified location is local for current node */ -bool fio_is_remote(fio_location location) +bool +fio_is_remote(fio_location location) { bool is_remote = MyLocation != FIO_LOCAL_HOST && location != FIO_LOCAL_HOST @@ -188,7 +218,8 @@ bool fio_is_remote(fio_location location) } /* Check if specified location is local for current node */ -bool fio_is_remote_simple(fio_location location) +bool +fio_is_remote_simple(fio_location location) { bool is_remote = MyLocation != FIO_LOCAL_HOST && location != FIO_LOCAL_HOST @@ -197,7 +228,8 @@ bool fio_is_remote_simple(fio_location location) } /* Try to read specified amount of bytes unless error or EOF are encountered */ -static ssize_t fio_read_all(int fd, void* buf, size_t size) +static ssize_t +fio_read_all(int fd, void* buf, size_t size) { size_t offs = 0; while (offs < size) @@ -219,7 +251,8 @@ static ssize_t fio_read_all(int fd, void* buf, size_t size) } /* Try to write specified amount of bytes unless error is encountered */ -static ssize_t fio_write_all(int fd, void const* buf, size_t size) +static ssize_t +fio_write_all(int fd, void const* buf, size_t size) { size_t offs = 0; while (offs < size) @@ -240,7 +273,8 @@ static ssize_t fio_write_all(int fd, void const* buf, size_t size) } /* Get version of remote agent */ -int fio_get_agent_version(void) +void +fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) { fio_header hdr; hdr.cop = FIO_AGENT_VERSION; @@ -248,12 +282,18 @@ int fio_get_agent_version(void) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size > payload_buf_size) + { + elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); + } - return hdr.arg; + *protocol = hdr.arg; + IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); } /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ -FILE* fio_open_stream(char const* path, fio_location location) +FILE* +fio_open_stream(char const* path, fio_location location) { FILE* f; if (fio_is_remote(location)) @@ -274,7 +314,7 @@ FILE* fio_open_stream(char const* path, fio_location location) IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); #ifdef WIN32 f = tmpfile(); - IO_CHECK(fwrite(f, 1, hdr.size, fio_stdin_buffer), hdr.size); + IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size); SYS_CHECK(fseek(f, 0, SEEK_SET)); #else f = fmemopen(fio_stdin_buffer, hdr.size, "r"); @@ -293,7 +333,8 @@ FILE* fio_open_stream(char const* path, fio_location location) } /* Close input stream */ -int fio_close_stream(FILE* f) +int +fio_close_stream(FILE* f) { if (fio_stdin_buffer) { @@ -304,7 +345,8 @@ int fio_close_stream(FILE* f) } /* Open directory */ -DIR* fio_opendir(char const* path, fio_location location) +DIR* +fio_opendir(char const* path, fio_location location) { DIR* dir; if (fio_is_remote(location)) @@ -345,7 +387,8 @@ DIR* fio_opendir(char const* path, fio_location location) } /* Get next directory entry */ -struct dirent* fio_readdir(DIR *dir) +struct dirent* +fio_readdir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) { @@ -373,7 +416,8 @@ struct dirent* fio_readdir(DIR *dir) } /* Close directory */ -int fio_closedir(DIR *dir) +int +fio_closedir(DIR *dir) { if (fio_is_remote_file((FILE*)dir)) { @@ -393,7 +437,8 @@ int fio_closedir(DIR *dir) } /* Open file */ -int fio_open(char const* path, int mode, fio_location location) +int +fio_open(char const* path, int mode, fio_location location) { int fd; if (fio_is_remote(location)) @@ -420,6 +465,7 @@ int fio_open(char const* path, int mode, fio_location location) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + /* check results */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); if (hdr.arg != 0) @@ -452,14 +498,17 @@ fio_disconnect(void) Assert(hdr.cop == FIO_DISCONNECTED); SYS_CHECK(close(fio_stdin)); SYS_CHECK(close(fio_stdout)); + SYS_CHECK(close(fio_stderr)); fio_stdin = 0; fio_stdout = 0; + fio_stderr = 0; wait_ssh(); } } /* Open stdio file */ -FILE* fio_fopen(char const* path, char const* mode, fio_location location) +FILE* +fio_fopen(char const* path, char const* mode, fio_location location) { FILE *f = NULL; @@ -504,7 +553,8 @@ FILE* fio_fopen(char const* path, char const* mode, fio_location location) } /* Format output to file stream */ -int fio_fprintf(FILE* f, char const* format, ...) +int +fio_fprintf(FILE* f, char const* format, ...) { int rc; va_list args; @@ -530,7 +580,8 @@ int fio_fprintf(FILE* f, char const* format, ...) } /* Flush stream data (does nothing for remote file) */ -int fio_fflush(FILE* f) +int +fio_fflush(FILE* f) { int rc = 0; if (!fio_is_remote_file(f)) @@ -539,13 +590,15 @@ int fio_fflush(FILE* f) } /* Sync file to the disk (does nothing for remote file) */ -int fio_flush(int fd) +int +fio_flush(int fd) { return fio_is_remote_fd(fd) ? 0 : fsync(fd); } /* Close output stream */ -int fio_fclose(FILE* f) +int +fio_fclose(FILE* f) { return fio_is_remote_file(f) ? fio_close(fio_fileno(f)) @@ -553,7 +606,8 @@ int fio_fclose(FILE* f) } /* Close file */ -int fio_close(int fd) +int +fio_close(int fd) { if (fio_is_remote_fd(fd)) { @@ -565,7 +619,15 @@ int fio_close(int fd) fio_fdset &= ~(1 << hdr.handle); IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - /* Note, that file is closed without waiting for confirmation */ + + /* Wait for response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } return 0; } @@ -575,16 +637,36 @@ int fio_close(int fd) } } +/* Close remote file implementation */ +static void +fio_close_impl(int fd, int out) +{ + fio_header hdr; + + hdr.cop = FIO_CLOSE; + hdr.arg = 0; + + if (close(fd) != 0) + hdr.arg = errno; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Truncate stdio file */ -int fio_ftruncate(FILE* f, off_t size) +int +fio_ftruncate(FILE* f, off_t size) { return fio_is_remote_file(f) ? fio_truncate(fio_fileno(f), size) : ftruncate(fileno(f), size); } -/* Truncate file */ -int fio_truncate(int fd, off_t size) +/* Truncate file + * TODO: make it synchronous + */ +int +fio_truncate(int fd, off_t size) { if (fio_is_remote_fd(fd)) { @@ -609,7 +691,8 @@ int fio_truncate(int fd, off_t size) /* * Read file from specified location. */ -int fio_pread(FILE* f, void* buf, off_t offs) +int +fio_pread(FILE* f, void* buf, off_t offs) { if (fio_is_remote_file(f)) { @@ -645,7 +728,8 @@ int fio_pread(FILE* f, void* buf, off_t offs) } /* Set position in stdio file */ -int fio_fseek(FILE* f, off_t offs) +int +fio_fseek(FILE* f, off_t offs) { return fio_is_remote_file(f) ? fio_seek(fio_fileno(f), offs) @@ -653,7 +737,9 @@ int fio_fseek(FILE* f, off_t offs) } /* Set position in file */ -int fio_seek(int fd, off_t offs) +/* TODO: make it synchronous or check async error */ +int +fio_seek(int fd, off_t offs) { if (fio_is_remote_fd(fd)) { @@ -674,16 +760,66 @@ int fio_seek(int fd, off_t offs) } } +/* seek is asynchronous */ +static void +fio_seek_impl(int fd, off_t offs) +{ + int rc; + + /* Quick exit for tainted agent */ + if (async_errormsg) + return; + + rc = lseek(fd, offs, SEEK_SET); + + if (rc < 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } +} + /* Write data to stdio file */ -size_t fio_fwrite(FILE* f, void const* buf, size_t size) +size_t +fio_fwrite(FILE* f, void const* buf, size_t size) { - return fio_is_remote_file(f) - ? fio_write(fio_fileno(f), buf, size) - : fwrite(buf, 1, size, f); + if (fio_is_remote_file(f)) + return fio_write(fio_fileno(f), buf, size); + else + return fwrite(buf, 1, size, f); } -/* Write data to the file */ -ssize_t fio_write(int fd, void const* buf, size_t size) +/* + * Write buffer to descriptor by calling write(), + * If size of written data is less than buffer size, + * then try to write what is left. + * We do this to get honest errno if there are some problems + * with filesystem, since writing less than buffer size + * is not considered an error. + */ +static ssize_t +durable_write(int fd, const char* buf, size_t size) +{ + off_t current_pos = 0; + size_t bytes_left = size; + + while (bytes_left > 0) + { + int rc = write(fd, buf + current_pos, bytes_left); + + if (rc <= 0) + return rc; + + bytes_left -= rc; + current_pos += rc; + } + + return size; +} + +/* Write data to the file synchronously */ +ssize_t +fio_write(int fd, void const* buf, size_t size) { if (fio_is_remote_fd(fd)) { @@ -696,45 +832,125 @@ ssize_t fio_write(int fd, void const* buf, size_t size) IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* set errno */ + if (hdr.arg > 0) + { + errno = hdr.arg; + return -1; + } + return size; } else { - return write(fd, buf, size); + return durable_write(fd, buf, size); + } +} + +static void +fio_write_impl(int fd, void const* buf, size_t size, int out) +{ + int rc; + fio_header hdr; + + rc = durable_write(fd, buf, size); + + hdr.arg = 0; + hdr.size = 0; + + if (rc < 0) + hdr.arg = errno; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + + return; +} + +size_t +fio_fwrite_async(FILE* f, void const* buf, size_t size) +{ + return fio_is_remote_file(f) + ? fio_write_async(fio_fileno(f), buf, size) + : fwrite(buf, 1, size, f); +} + +/* Write data to the file */ +/* TODO: support async report error */ +ssize_t +fio_write_async(int fd, void const* buf, size_t size) +{ + if (size == 0) + return 0; + + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_WRITE_ASYNC; + hdr.handle = fd & ~FIO_PIPE_MARKER; + hdr.size = size; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + } + else + return durable_write(fd, buf, size); + + return size; +} + +static void +fio_write_async_impl(int fd, void const* buf, size_t size, int out) +{ + /* Quick exit if agent is tainted */ + if (async_errormsg) + return; + + if (durable_write(fd, buf, size) <= 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); } } int32 -fio_decompress(void* dst, void const* src, size_t size, int compress_alg) +fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg) { - const char *errormsg = NULL; + const char *internal_errormsg = NULL; int32 uncompressed_size = do_decompress(dst, BLCKSZ, src, size, - compress_alg, &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) + compress_alg, &internal_errormsg); + + if (uncompressed_size < 0 && internal_errormsg != NULL) { - elog(WARNING, "An error occured during decompressing block: %s", errormsg); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, "An error occured during decompressing block: %s", internal_errormsg); return -1; } if (uncompressed_size != BLCKSZ) { - elog(ERROR, "Page uncompressed to %d bytes != BLCKSZ", - uncompressed_size); + *errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(*errormsg, ERRMSG_MAX_LEN, "Page uncompressed to %d bytes != BLCKSZ", uncompressed_size); return -1; } return uncompressed_size; } /* Write data to the file */ -ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg) +ssize_t +fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) { if (fio_is_remote_file(f)) { fio_header hdr; - hdr.cop = FIO_WRITE_COMPRESSED; + hdr.cop = FIO_WRITE_COMPRESSED_ASYNC; hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; hdr.size = size; hdr.arg = compress_alg; @@ -746,25 +962,127 @@ ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compres } else { - char uncompressed_buf[BLCKSZ]; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); + char *errormsg = NULL; + char decompressed_buf[BLCKSZ]; + int32 decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &errormsg); + + if (decompressed_size < 0) + elog(ERROR, "%s", errormsg); - return (uncompressed_size < 0) - ? uncompressed_size - : fwrite(uncompressed_buf, 1, uncompressed_size, f); + return fwrite(decompressed_buf, 1, decompressed_size, f); } } -static ssize_t +static void fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) { - char uncompressed_buf[BLCKSZ]; - int32 uncompressed_size = fio_decompress(uncompressed_buf, buf, size, compress_alg); - return fio_write_all(fd, uncompressed_buf, uncompressed_size); + int32 decompressed_size; + char decompressed_buf[BLCKSZ]; + + /* If the previous command already have failed, + * then there is no point in bashing a head against the wall + */ + if (async_errormsg) + return; + + /* decompress chunk */ + decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &async_errormsg); + + if (decompressed_size < 0) + return; + + if (durable_write(fd, decompressed_buf, decompressed_size) <= 0) + { + async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); + snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); + } +} + +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_file(FILE* f, char **errmsg) +{ + if (fio_is_remote_file(f)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + + return 0; +} + +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_fd(int fd, char **errmsg) +{ + if (fio_is_remote_fd(fd)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + return 0; +} + +static void +fio_get_async_error_impl(int out) +{ + fio_header hdr; + hdr.cop = FIO_GET_ASYNC_ERROR; + + /* send error message */ + if (async_errormsg) + { + hdr.size = strlen(async_errormsg) + 1; + + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* send message itself */ + IO_CHECK(fio_write_all(out, async_errormsg, hdr.size), hdr.size); + + //TODO: should we reset the tainted state ? +// pg_free(async_errormsg); +// async_errormsg = NULL; + } + else + { + hdr.size = 0; + /* send header */ + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } } /* Read data from stdio file */ -ssize_t fio_fread(FILE* f, void* buf, size_t size) +ssize_t +fio_fread(FILE* f, void* buf, size_t size) { size_t rc; if (fio_is_remote_file(f)) @@ -774,7 +1092,8 @@ ssize_t fio_fread(FILE* f, void* buf, size_t size) } /* Read data from file */ -ssize_t fio_read(int fd, void* buf, size_t size) +ssize_t +fio_read(int fd, void* buf, size_t size) { if (fio_is_remote_fd(fd)) { @@ -800,7 +1119,8 @@ ssize_t fio_read(int fd, void* buf, size_t size) } /* Get information about file */ -int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) +int +fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) { if (fio_is_remote(location)) { @@ -832,8 +1152,87 @@ int fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_locatio } } +/* + * Compare, that filename1 and filename2 is the same file + * in windows compare only filenames + */ +bool +fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) +{ + char *abs_name1 = make_absolute_path(filename1); + char *abs_name2 = make_absolute_path(filename2); + bool result = strcmp(abs_name1, abs_name2) == 0; + +#ifndef WIN32 + if (!result) + { + struct stat stat1, stat2; + + if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) + { + if (errno == ENOENT) + return false; + elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); + } + + if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) + { + if (errno == ENOENT) + return false; + elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); + } + + result = (stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev); + } +#endif + free(abs_name2); + free(abs_name1); + return result; +} + +/* + * Read value of a symbolic link + * this is a wrapper about readlink() syscall + * side effects: string truncation occur (and it + * can be checked by caller by comparing + * returned value >= valsiz) + */ +ssize_t +fio_readlink(const char *path, char *value, size_t valsiz, fio_location location) +{ + if (!fio_is_remote(location)) + { + /* readlink don't place trailing \0 */ + ssize_t len = readlink(path, value, valsiz); + value[len < valsiz ? len : valsiz] = '\0'; + return len; + } + else + { + fio_header hdr; + size_t path_len = strlen(path) + 1; + + hdr.cop = FIO_READLINK; + hdr.handle = -1; + Assert(valsiz <= UINT_MAX); /* max value of fio_header.arg */ + hdr.arg = valsiz; + hdr.size = path_len; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_READLINK); + Assert(hdr.size <= valsiz); + IO_CHECK(fio_read_all(fio_stdin, value, hdr.size), hdr.size); + value[hdr.size < valsiz ? hdr.size : valsiz] = '\0'; + return hdr.size; + } +} + /* Check presence of the file */ -int fio_access(char const* path, int mode, fio_location location) +int +fio_access(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -864,7 +1263,8 @@ int fio_access(char const* path, int mode, fio_location location) } /* Create symbolic link */ -int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) +int +fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) { if (fio_is_remote(location)) { @@ -891,7 +1291,8 @@ int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_l } } -static void fio_symlink_impl(int out, char *buf, bool overwrite) +static void +fio_symlink_impl(int out, char *buf, bool overwrite) { char *linked_path = buf; char *link_path = buf + strlen(buf) + 1; @@ -905,7 +1306,8 @@ static void fio_symlink_impl(int out, char *buf, bool overwrite) } /* Rename file */ -int fio_rename(char const* old_path, char const* new_path, fio_location location) +int +fio_rename(char const* old_path, char const* new_path, fio_location location) { if (fio_is_remote(location)) { @@ -931,7 +1333,8 @@ int fio_rename(char const* old_path, char const* new_path, fio_location location } /* Sync file to disk */ -int fio_sync(char const* path, fio_location location) +int +fio_sync(char const* path, fio_location location) { if (fio_is_remote(location)) { @@ -972,9 +1375,20 @@ int fio_sync(char const* path, fio_location location) } } +enum { + GET_CRC32_DECOMPRESS = 1, + GET_CRC32_MISSING_OK = 2, + GET_CRC32_TRUNCATED = 4 +}; + /* Get crc32 of file */ -pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress) +static pg_crc32 +fio_get_crc32_ex(const char *file_path, fio_location location, + bool decompress, bool missing_ok, bool truncated) { + if (decompress && truncated) + elog(ERROR, "Could not calculate CRC for compressed truncated file"); + if (fio_is_remote(location)) { fio_header hdr; @@ -986,7 +1400,11 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decomp hdr.arg = 0; if (decompress) - hdr.arg = 1; + hdr.arg = GET_CRC32_DECOMPRESS; + if (missing_ok) + hdr.arg |= GET_CRC32_MISSING_OK; + if (truncated) + hdr.arg |= GET_CRC32_TRUNCATED; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); @@ -997,14 +1415,31 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decomp else { if (decompress) - return pgFileGetCRCgz(file_path, true, true); + return pgFileGetCRCgz(file_path, true, missing_ok); + else if (truncated) + return pgFileGetCRCTruncated(file_path, true, missing_ok); else - return pgFileGetCRC(file_path, true, true); + return pgFileGetCRC(file_path, true, missing_ok); } } +pg_crc32 +fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok) +{ + return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false); +} + +pg_crc32 +fio_get_crc32_truncated(const char *file_path, fio_location location, + bool missing_ok) +{ + return fio_get_crc32_ex(file_path, location, false, missing_ok, true); +} + /* Remove file */ -int fio_unlink(char const* path, fio_location location) +int +fio_unlink(char const* path, fio_location location) { if (fio_is_remote(location)) { @@ -1026,8 +1461,11 @@ int fio_unlink(char const* path, fio_location location) } } -/* Create directory */ -int fio_mkdir(char const* path, int mode, fio_location location) +/* Create directory + * TODO: add strict flag + */ +int +fio_mkdir(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -1048,12 +1486,13 @@ int fio_mkdir(char const* path, int mode, fio_location location) } else { - return dir_create_dir(path, mode); + return dir_create_dir(path, mode, false); } } /* Change file mode */ -int fio_chmod(char const* path, int mode, fio_location location) +int +fio_chmod(char const* path, int mode, fio_location location) { if (fio_is_remote(location)) { @@ -1080,6 +1519,11 @@ int fio_chmod(char const* path, int mode, fio_location location) #define ZLIB_BUFFER_SIZE (64*1024) #define MAX_WBITS 15 /* 32K LZ77 window */ #define DEF_MEM_LEVEL 8 +/* last bit used to differenciate remote gzFile from local gzFile + * TODO: this is insane, we should create our own scructure for this, + * not flip some bits in someone's else and hope that it will not break + * between zlib versions. + */ #define FIO_GZ_REMOTE_MARKER 1 typedef struct fioGZFile @@ -1092,6 +1536,32 @@ typedef struct fioGZFile Bytef buf[ZLIB_BUFFER_SIZE]; } fioGZFile; +/* check if remote agent encountered any error during execution of async operations */ +int +fio_check_error_fd_gz(gzFile f, char **errmsg) +{ + if (f && ((size_t)f & FIO_GZ_REMOTE_MARKER)) + { + fio_header hdr; + + hdr.cop = FIO_GET_ASYNC_ERROR; + hdr.size = 0; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + + /* check results */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (hdr.size > 0) + { + *errmsg = pgut_malloc(ERRMSG_MAX_LEN); + IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); + return 1; + } + } + return 0; +} + /* On error returns NULL and errno should be checked */ gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location) @@ -1264,7 +1734,7 @@ fio_gzwrite(gzFile f, void const* buf, unsigned size) break; } } - rc = fio_write(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); + rc = fio_write_async(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); if (rc >= 0) { gz->strm.next_out += rc; @@ -1317,7 +1787,8 @@ fio_gzclose(gzFile f) } } -int fio_gzeof(gzFile f) +int +fio_gzeof(gzFile f) { if ((size_t)f & FIO_GZ_REMOTE_MARKER) { @@ -1330,7 +1801,8 @@ int fio_gzeof(gzFile f) } } -const char* fio_gzerror(gzFile f, int *errnum) +const char* +fio_gzerror(gzFile f, int *errnum) { if ((size_t)f & FIO_GZ_REMOTE_MARKER) { @@ -1345,7 +1817,8 @@ const char* fio_gzerror(gzFile f, int *errnum) } } -z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) +z_off_t +fio_gzseek(gzFile f, z_off_t offset, int whence) { Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); return gzseek(f, offset, whence); @@ -1357,7 +1830,8 @@ z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence) /* Send file content * Note: it should not be used for large files. */ -static void fio_load_file(int out, char const* path) +static void +fio_load_file(int out, char const* path) { int fd = open(path, O_RDONLY); fio_header hdr; @@ -1399,10 +1873,11 @@ static void fio_load_file(int out, char const* path) * In case of DELTA mode horizonLsn must be a valid lsn, * otherwise it should be set to InvalidXLogRecPtr. */ -int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, - int calg, int clevel, uint32 checksum_version, - datapagemap_t *pagemap, BlockNumber* err_blknum, - char **errormsg, BackupPageHeader2 **headers) +int +fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber* err_blknum, char **errormsg, + BackupPageHeader2 **headers) { FILE *out = NULL; char *out_buf = NULL; @@ -1415,7 +1890,7 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f /* send message with header - 8bytes 24bytes var var + 16bytes 24bytes var var -------------------------------------------------------------- | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | -------------------------------------------------------------- @@ -1423,10 +1898,10 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f req.hdr.cop = FIO_SEND_PAGES; - if (pagemap) + if (use_pagemap) { - req.hdr.size = sizeof(fio_send_request) + pagemap->bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = pagemap->bitmapsize; + req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; + req.arg.bitmapsize = (*file).pagemap.bitmapsize; /* TODO: add optimization for the case of pagemap * containing small number of blocks with big serial numbers: @@ -1465,8 +1940,8 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); /* send pagemap if any */ - if (pagemap) - IO_CHECK(fio_write_all(fio_stdout, pagemap->bitmap, pagemap->bitmapsize), pagemap->bitmapsize); + if (use_pagemap) + IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); while (true) { @@ -1549,6 +2024,197 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f return n_blocks_read; } +/* + * Return number of actually(!) readed blocks, attempts or + * half-readed block are not counted. + * Return values in case of error: + * FILE_MISSING + * OPEN_FAILED + * READ_ERROR + * PAGE_CORRUPTION + * WRITE_FAILED + * + * If none of the above, this function return number of blocks + * readed by remote agent. + * + * In case of DELTA mode horizonLsn must be a valid lsn, + * otherwise it should be set to InvalidXLogRecPtr. + * Взято из fio_send_pages + */ +int +fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, + XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, + bool use_pagemap, BlockNumber* err_blknum, char **errormsg) +{ + FILE *out = NULL; + char *out_buf = NULL; + struct { + fio_header hdr; + fio_send_request arg; + } req; + BlockNumber n_blocks_read = 0; + BlockNumber blknum = 0; + + /* send message with header + + 16bytes 24bytes var var + -------------------------------------------------------------- + | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | + -------------------------------------------------------------- + */ + + req.hdr.cop = FIO_SEND_PAGES; + + if (use_pagemap) + { + req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; + req.arg.bitmapsize = (*file).pagemap.bitmapsize; + + /* TODO: add optimization for the case of pagemap + * containing small number of blocks with big serial numbers: + * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 + */ + } + else + { + req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; + req.arg.bitmapsize = 0; + } + + req.arg.nblocks = file->size/BLCKSZ; + req.arg.segmentno = file->segno * RELSEG_SIZE; + req.arg.horizonLsn = horizonLsn; + req.arg.checksumVersion = checksum_version; + req.arg.calg = calg; + req.arg.clevel = clevel; + req.arg.path_len = strlen(from_fullpath) + 1; + + file->compress_alg = calg; /* TODO: wtf? why here? */ + +//<----- +// datapagemap_iterator_t *iter; +// BlockNumber blkno; +// iter = datapagemap_iterate(pagemap); +// while (datapagemap_next(iter, &blkno)) +// elog(INFO, "block %u", blkno); +// pg_free(iter); +//<----- + + /* send header */ + IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); + + /* send file path */ + IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); + + /* send pagemap if any */ + if (use_pagemap) + IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); + + out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + if (out == NULL) + elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); + + /* update file permission */ + if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) + elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, + strerror(errno)); + + elog(VERBOSE, "ftruncate file \"%s\" to size %lu", + to_fullpath, file->size); + if (fio_ftruncate(out, file->size) == -1) + elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", + to_fullpath, file->size, strerror(errno)); + + if (!fio_is_remote_file(out)) + { + out_buf = pgut_malloc(STDIO_BUFSIZE); + setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); + } + + while (true) + { + fio_header hdr; + char buf[BLCKSZ + sizeof(BackupPageHeader)]; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + + if (interrupted) + elog(ERROR, "Interrupted during page reading"); + + if (hdr.cop == FIO_ERROR) + { + /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); + } + + return hdr.arg; + } + else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) + { + *err_blknum = hdr.arg; + + if (hdr.size > 0) + { + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + *errormsg = pgut_malloc(hdr.size); + snprintf(*errormsg, hdr.size, "%s", buf); + } + return PAGE_CORRUPTION; + } + else if (hdr.cop == FIO_SEND_FILE_EOF) + { + /* n_blocks_read reported by EOF */ + n_blocks_read = hdr.arg; + + /* receive headers if any */ + if (hdr.size > 0) + { + char *tmp = pgut_malloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, tmp, hdr.size), hdr.size); + pg_free(tmp); + } + + break; + } + else if (hdr.cop == FIO_PAGE) + { + blknum = hdr.arg; + + Assert(hdr.size <= sizeof(buf)); + IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); + + COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + + if (fio_fseek(out, blknum * BLCKSZ) < 0) + { + elog(ERROR, "Cannot seek block %u of \"%s\": %s", + blknum, to_fullpath, strerror(errno)); + } + // должен прилетать некомпрессированный блок с заголовком + // Вставить assert? + if (fio_fwrite(out, buf + sizeof(BackupPageHeader), hdr.size - sizeof(BackupPageHeader)) != BLCKSZ) + { + fio_fclose(out); + *err_blknum = blknum; + return WRITE_FAILED; + } + file->write_size += BLCKSZ; + file->uncompressed_size += BLCKSZ; + } + else + elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); + } + + if (out) + fclose(out); + pg_free(out_buf); + + return n_blocks_read; +} + /* TODO: read file using large buffer * Return codes: * FIO_ERROR: @@ -1559,7 +2225,8 @@ int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *f * FIO_SEND_FILE_CORRUPTION * FIO_SEND_FILE_EOF */ -static void fio_send_pages_impl(int out, char* buf) +static void +fio_send_pages_impl(int out, char* buf) { FILE *in = NULL; BlockNumber blknum = 0; @@ -1729,13 +2396,13 @@ static void fio_send_pages_impl(int out, char* buf) n_blocks_read++; /* - * horizonLsn is not 0 only in case of delta backup. + * horizonLsn is not 0 only in case of delta and ptrack backup. * As far as unsigned number are always greater or equal than zero, * there is no sense to add more checks. */ - if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page, ptrack */ + if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page */ (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta */ + (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta, ptrack */ { int compressed_size = 0; char write_buffer[BLCKSZ*2]; @@ -1829,7 +2496,8 @@ static void fio_send_pages_impl(int out, char* buf) * ZLIB_ERROR (-5) * REMOTE_ERROR (-6) */ -int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg) +int +fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; @@ -1869,11 +2537,22 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o exit_code = hdr.arg; goto cleanup; } - else if (hdr.cop == FIO_PAGE) + else if (hdr.cop == FIO_PAGE || hdr.cop == FIO_PAGE_ZERO) { int rc; - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + unsigned size; + if (hdr.cop == FIO_PAGE) + { + Assert(hdr.size <= CHUNK_SIZE); + size = hdr.size; + IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); + } + else + { + Assert(hdr.arg <= CHUNK_SIZE); + size = hdr.arg; + memset(in_buf, 0, hdr.arg); + } /* We have received a chunk of compressed data, lets decompress it */ if (strm == NULL) @@ -1884,7 +2563,7 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o /* The fields next_in, avail_in initialized before init */ strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; + strm->avail_in = size; rc = inflateInit2(strm, 15 + 16); @@ -1901,7 +2580,7 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o else { strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; + strm->avail_in = size; } strm->next_out = (Bytef *)out_buf; /* output buffer */ @@ -1978,6 +2657,113 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o return exit_code; } +typedef struct send_file_state { + bool calc_crc; + uint32_t crc; + int64_t read_size; + int64_t write_size; +} send_file_state; + +/* find page border of all-zero tail */ +static size_t +find_zero_tail(char *buf, size_t len) +{ + size_t i, l; + size_t granul = sizeof(zerobuf); + + if (len == 0) + return 0; + + /* fast check for last bytes */ + l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY); + i = len - l; + if (memcmp(buf + i, zerobuf, l) != 0) + return len; + + /* coarse search for zero tail */ + i = (len-1) & ~(granul-1); + l = len - i; + for (;;) + { + if (memcmp(buf+i, zerobuf, l) != 0) + { + i += l; + break; + } + if (i == 0) + break; + i -= granul; + l = granul; + } + + len = i; + /* search zero tail with finer granularity */ + for (granul = sizeof(zerobuf)/2; + len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY; + granul /= 2) + { + if (granul > l) + continue; + i = (len-1) & ~(granul-1); + l = len - i; + if (memcmp(buf+i, zerobuf, l) == 0) + len = i; + } + + return len; +} + +static void +fio_send_file_crc(send_file_state* st, char *buf, size_t len) +{ + int64_t write_size; + + if (!st->calc_crc) + return; + + write_size = st->write_size; + while (st->read_size > write_size) + { + size_t crc_len = Min(st->read_size - write_size, sizeof(zerobuf)); + COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len); + write_size += crc_len; + } + + if (len > 0) + COMP_FILE_CRC32(true, st->crc, buf, len); +} + +static bool +fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len) +{ + if (len == 0) + return true; + +#ifdef WIN32 + if (st->read_size > st->write_size && + _chsize_s(fileno(out), st->read_size) != 0) + { + elog(WARNING, "Could not change file size to %lld: %m", st->read_size); + return false; + } +#endif + if (st->read_size > st->write_size && + fseeko(out, st->read_size, SEEK_SET) != 0) + { + return false; + } + + if (fwrite(buf, 1, len, out) != len) + { + return false; + } + + st->read_size += len; + st->write_size = st->read_size; + + return true; +} + /* Receive chunks of data and write them to destination file. * Return codes: * SEND_OK (0) @@ -1989,13 +2775,23 @@ int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* o * OPEN_FAILED and READ_FAIL should also set errormsg. * If pgFile is not NULL then we must calculate crc and read_size for it. */ -int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, +int +fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, pgFile *file, char **errormsg) { fio_header hdr; int exit_code = SEND_OK; size_t path_len = strlen(from_fullpath) + 1; char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */ + send_file_state st = {false, 0, 0, 0}; + + memset(&hdr, 0, sizeof(hdr)); + + if (file) + { + st.calc_crc = true; + st.crc = file->crc; + } hdr.cop = FIO_SEND_FILE; hdr.size = path_len; @@ -2013,6 +2809,37 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, if (hdr.cop == FIO_SEND_FILE_EOF) { + if (st.write_size < st.read_size) + { + if (!cut_zero_tail) + { + /* + * We still need to calc crc for zero tail. + */ + fio_send_file_crc(&st, NULL, 0); + + /* + * Let's write single zero byte to the end of file to restore + * logical size. + * Well, it would be better to use ftruncate here actually, + * but then we need to change interface. + */ + st.read_size -= 1; + buf[0] = 0; + if (!fio_send_file_write(out, &st, buf, 1)) + { + exit_code = WRITE_FAILED; + break; + } + } + } + + if (file) + { + file->crc = st.crc; + file->read_size = st.read_size; + file->write_size = st.write_size; + } break; } else if (hdr.cop == FIO_ERROR) @@ -2033,17 +2860,23 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); /* We have received a chunk of data data, lets write it out */ - if (fwrite(buf, 1, hdr.size, out) != hdr.size) + fio_send_file_crc(&st, buf, hdr.size); + if (!fio_send_file_write(out, &st, buf, hdr.size)) { exit_code = WRITE_FAILED; break; } + } + else if (hdr.cop == FIO_PAGE_ZERO) + { + Assert(hdr.size == 0); + Assert(hdr.arg <= CHUNK_SIZE); - if (file) - { - file->read_size += hdr.size; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - } + /* + * We have received a chunk of zero data, lets just think we + * wrote it. + */ + st.read_size += hdr.arg; } else { @@ -2059,6 +2892,128 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, return exit_code; } +int +fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, + pgFile *file, char **errormsg) +{ + FILE* in; + char* buf; + size_t read_len, non_zero_len; + int exit_code = SEND_OK; + send_file_state st = {false, 0, 0, 0}; + + if (file) + { + st.calc_crc = true; + st.crc = file->crc; + } + + /* open source file for read */ + in = fopen(from_fullpath, PG_BINARY_R); + if (in == NULL) + { + /* maybe deleted, it's not error in case of backup */ + if (errno == ENOENT) + return FILE_MISSING; + + + *errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath, + strerror(errno)); + return OPEN_FAILED; + } + + /* disable stdio buffering for local input/output files to avoid triple buffering */ + setvbuf(in, NULL, _IONBF, BUFSIZ); + setvbuf(out, NULL, _IONBF, BUFSIZ); + + /* allocate 64kB buffer */ + buf = pgut_malloc(CHUNK_SIZE); + + /* copy content and calc CRC */ + for (;;) + { + read_len = fread(buf, 1, CHUNK_SIZE, in); + + if (ferror(in)) + { + *errormsg = psprintf("Cannot read from file \"%s\": %s", + from_fullpath, strerror(errno)); + exit_code = READ_FAILED; + goto cleanup; + } + + if (read_len > 0) + { + non_zero_len = find_zero_tail(buf, read_len); + /* + * It is dirty trick to silence warnings in CFS GC process: + * backup at least cfs header size bytes. + */ + if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + st.read_size + read_len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + st.read_size + read_len); + non_zero_len -= st.read_size; + } + if (non_zero_len > 0) + { + fio_send_file_crc(&st, buf, non_zero_len); + if (!fio_send_file_write(out, &st, buf, non_zero_len)) + { + exit_code = WRITE_FAILED; + goto cleanup; + } + } + if (non_zero_len < read_len) + { + /* Just pretend we wrote it. */ + st.read_size += read_len - non_zero_len; + } + } + + if (feof(in)) + break; + } + + if (st.write_size < st.read_size) + { + if (!cut_zero_tail) + { + /* + * We still need to calc crc for zero tail. + */ + fio_send_file_crc(&st, NULL, 0); + + /* + * Let's write single zero byte to the end of file to restore + * logical size. + * Well, it would be better to use ftruncate here actually, + * but then we need to change interface. + */ + st.read_size -= 1; + buf[0] = 0; + if (!fio_send_file_write(out, &st, buf, 1)) + { + exit_code = WRITE_FAILED; + goto cleanup; + } + } + } + + if (file) + { + file->crc = st.crc; + file->read_size = st.read_size; + file->write_size = st.write_size; + } + +cleanup: + free(buf); + fclose(in); + return exit_code; +} + /* Send file content * On error we return FIO_ERROR message with following codes * FIO_ERROR: @@ -2070,12 +3025,14 @@ int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, * FIO_SEND_FILE_EOF * */ -static void fio_send_file_impl(int out, char const* path) +static void +fio_send_file_impl(int out, char const* path) { FILE *fp; fio_header hdr; char *buf = pgut_malloc(CHUNK_SIZE); size_t read_len = 0; + int64_t read_size = 0; char *errormsg = NULL; /* open source file for read */ @@ -2118,6 +3075,7 @@ static void fio_send_file_impl(int out, char const* path) for (;;) { read_len = fread(buf, 1, CHUNK_SIZE, fp); + memset(&hdr, 0, sizeof(hdr)); /* report error */ if (ferror(fp)) @@ -2138,10 +3096,36 @@ static void fio_send_file_impl(int out, char const* path) if (read_len > 0) { /* send chunk */ - hdr.cop = FIO_PAGE; - hdr.size = read_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, read_len), read_len); + int64_t non_zero_len = find_zero_tail(buf, read_len); + /* + * It is dirty trick to silence warnings in CFS GC process: + * backup at least cfs header size bytes. + */ + if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + read_size + read_len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + read_size + read_len); + non_zero_len -= read_size; + } + + if (non_zero_len > 0) + { + hdr.cop = FIO_PAGE; + hdr.size = non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len); + } + + if (non_zero_len < read_len) + { + hdr.cop = FIO_PAGE_ZERO; + hdr.size = 0; + hdr.arg = read_len - non_zero_len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + } + + read_size += read_len; } if (feof(fp)) @@ -2160,10 +3144,215 @@ static void fio_send_file_impl(int out, char const* path) return; } +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + */ +pg_crc32 +pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) +{ + FILE *fp; + pg_crc32 crc = 0; + char *buf; + size_t len = 0; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + + if (feof(fp)) + break; + } + + FIN_FILE_CRC32(use_crc32c, crc); + fclose(fp); + pg_free(buf); + + return crc; +} + +/* + * Read the local file to compute CRC for it extened to real_size. + */ +pg_crc32 +pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok) +{ + FILE *fp; + char *buf; + size_t len = 0; + size_t non_zero_len; + send_file_state st = {true, 0, 0, 0}; + + INIT_FILE_CRC32(use_crc32c, st.crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, st.crc); + return st.crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(CHUNK_SIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + non_zero_len = find_zero_tail(buf, len); + /* same trick as in fio_send_file */ + if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && + st.read_size + len > 0) + { + non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, + st.read_size + len); + non_zero_len -= st.read_size; + } + if (non_zero_len) + { + fio_send_file_crc(&st, buf, non_zero_len); + st.write_size += st.read_size + non_zero_len; + } + st.read_size += len; + + if (feof(fp)) + break; + } + + FIN_FILE_CRC32(use_crc32c, st.crc); + fclose(fp); + pg_free(buf); + + return st.crc; +} + +/* + * Read the local file to compute its CRC. + * We cannot make decision about file decompression because + * user may ask to backup already compressed files and we should be + * obvious about it. + */ +pg_crc32 +pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +{ + gzFile fp; + pg_crc32 crc = 0; + int len = 0; + int err; + char *buf; + + INIT_FILE_CRC32(use_crc32c, crc); + + /* open file in binary read mode */ + fp = gzopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (errno == ENOENT) + { + if (missing_ok) + { + FIN_FILE_CRC32(use_crc32c, crc); + return crc; + } + } + + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); + } + + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + for (;;) + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = gzread(fp, buf, STDIO_BUFSIZE); + + if (len <= 0) + { + /* we either run into eof or error */ + if (gzeof(fp)) + break; + else + { + const char *err_str = NULL; + + err_str = gzerror(fp, &err); + elog(ERROR, "Cannot read from compressed file %s", err_str); + } + } + + /* update CRC */ + COMP_FILE_CRC32(use_crc32c, crc, buf, len); + } + + FIN_FILE_CRC32(use_crc32c, crc); + gzclose(fp); + pg_free(buf); + + return crc; +} + /* Compile the array of files located on remote machine in directory root */ -void fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) +static void +fio_list_dir_internal(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, bool backup_logs, + bool skip_hidden, int external_dir_num) { fio_header hdr; fio_list_dir_request req; @@ -2211,7 +3400,6 @@ void fio_list_dir(parray *files, const char *root, bool exclude, file->size = fio_file.size; file->mtime = fio_file.mtime; file->is_datafile = fio_file.is_datafile; - file->is_database = fio_file.is_database; file->tblspcOid = fio_file.tblspcOid; file->dbOid = fio_file.dbOid; file->relOid = fio_file.relOid; @@ -2254,7 +3442,8 @@ void fio_list_dir(parray *files, const char *root, bool exclude, * * TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages */ -static void fio_list_dir_impl(int out, char* buf) +static void +fio_list_dir_impl(int out, char* buf) { int i; fio_header hdr; @@ -2284,7 +3473,6 @@ static void fio_list_dir_impl(int out, char* buf) fio_file.size = file->size; fio_file.mtime = file->mtime; fio_file.is_datafile = file->is_datafile; - fio_file.is_database = file->is_database; fio_file.tblspcOid = file->tblspcOid; fio_file.dbOid = file->dbOid; fio_file.relOid = file->relOid; @@ -2319,6 +3507,20 @@ static void fio_list_dir_impl(int out, char* buf) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } +/* Wrapper for directory listing */ +void +fio_list_dir(parray *files, const char *root, bool exclude, + bool follow_symlink, bool add_root, bool backup_logs, + bool skip_hidden, int external_dir_num) +{ + if (fio_is_remote(FIO_DB_HOST)) + fio_list_dir_internal(files, root, exclude, follow_symlink, add_root, + backup_logs, skip_hidden, external_dir_num); + else + dir_list_file(files, root, exclude, follow_symlink, add_root, + backup_logs, skip_hidden, external_dir_num, FIO_LOCAL_HOST); +} + PageState * fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) @@ -2362,7 +3564,8 @@ fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks } } -static void fio_get_checksum_map_impl(int out, char *buf) +static void +fio_get_checksum_map_impl(int out, char *buf) { fio_header hdr; PageState *checksum_map = NULL; @@ -2429,7 +3632,8 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version, return lsn_map; } -static void fio_get_lsn_map_impl(int out, char *buf) +static void +fio_get_lsn_map_impl(int out, char *buf) { fio_header hdr; datapagemap_t *lsn_map = NULL; @@ -2455,11 +3659,60 @@ static void fio_get_lsn_map_impl(int out, char *buf) } } +/* + * Return pid of postmaster process running in given pgdata on local machine. + * Return 0 if there is none. + * Return 1 if postmaster.pid is mangled. + */ +static pid_t +local_check_postmaster(const char *pgdata) +{ + FILE *fp; + pid_t pid; + char pid_file[MAXPGPATH]; + + join_path_components(pid_file, pgdata, "postmaster.pid"); + + fp = fopen(pid_file, "r"); + if (fp == NULL) + { + /* No pid file, acceptable*/ + if (errno == ENOENT) + return 0; + else + elog(ERROR, "Cannot open file \"%s\": %s", + pid_file, strerror(errno)); + } + + if (fscanf(fp, "%i", &pid) != 1) + { + /* something is wrong with the file content */ + pid = 1; + } + + if (pid > 1) + { + if (kill(pid, 0) != 0) + { + /* process no longer exists */ + if (errno == ESRCH) + pid = 0; + else + elog(ERROR, "Failed to send signal 0 to a process %d: %s", + pid, strerror(errno)); + } + } + + fclose(fp); + return pid; +} + /* * Go to the remote host and get postmaster pid from file postmaster.pid * and check that process is running, if process is running, return its pid number. */ -pid_t fio_check_postmaster(const char *pgdata, fio_location location) +pid_t +fio_check_postmaster(const char *pgdata, fio_location location) { if (fio_is_remote(location)) { @@ -2476,16 +3729,17 @@ pid_t fio_check_postmaster(const char *pgdata, fio_location location) return hdr.arg; } else - return check_postmaster(pgdata); + return local_check_postmaster(pgdata); } -static void fio_check_postmaster_impl(int out, char *buf) +static void +fio_check_postmaster_impl(int out, char *buf) { fio_header hdr; pid_t postmaster_pid; char *pgdata = (char*) buf; - postmaster_pid = check_postmaster(pgdata); + postmaster_pid = local_check_postmaster(pgdata); /* send arrays of checksums to main process */ hdr.arg = postmaster_pid; @@ -2524,7 +3778,8 @@ fio_delete_impl(mode_t mode, char *buf) } /* Execute commands at remote host */ -void fio_communicate(int in, int out) +void +fio_communicate(int in, int out) { /* * Map of file and directory descriptors. @@ -2557,6 +3812,7 @@ void fio_communicate(int in, int out) } IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); } + errno = 0; /* reset errno */ switch (hdr.cop) { case FIO_LOAD: /* Send file content */ fio_load_file(out, buf); @@ -2592,13 +3848,17 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CLOSE: /* Close file */ - SYS_CHECK(close(fd[hdr.handle])); + fio_close_impl(fd[hdr.handle], out); break; case FIO_WRITE: /* Write to the current position in file */ - IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); +// IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); + fio_write_impl(fd[hdr.handle], buf, hdr.size, out); + break; + case FIO_WRITE_ASYNC: /* Write to the current position in file */ + fio_write_async_impl(fd[hdr.handle], buf, hdr.size, out); break; - case FIO_WRITE_COMPRESSED: /* Write to the current position in file */ - IO_CHECK(fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg), BLCKSZ); + case FIO_WRITE_COMPRESSED_ASYNC: /* Write to the current position in file */ + fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg); break; case FIO_READ: /* Read from the current position in file */ if ((size_t)hdr.arg > buf_size) { @@ -2622,9 +3882,16 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); break; case FIO_AGENT_VERSION: - hdr.arg = AGENT_PROTOCOL_VERSION; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; + { + size_t payload_size = prepare_compatibility_str(buf, buf_size); + + hdr.arg = AGENT_PROTOCOL_VERSION; + hdr.size = payload_size; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); + break; + } case FIO_STAT: /* Get information about file with specified path */ hdr.size = sizeof(st); rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); @@ -2648,14 +3915,14 @@ void fio_communicate(int in, int out) break; case FIO_MKDIR: /* Create directory */ hdr.size = 0; - hdr.arg = dir_create_dir(buf, hdr.arg); + hdr.arg = dir_create_dir(buf, hdr.arg, false); IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_CHMOD: /* Change file mode */ SYS_CHECK(chmod(buf, hdr.arg)); break; case FIO_SEEK: /* Set current position in file */ - SYS_CHECK(lseek(fd[hdr.handle], hdr.arg, SEEK_SET)); + fio_seek_impl(fd[hdr.handle], hdr.arg); break; case FIO_TRUNCATE: /* Truncate file */ SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); @@ -2687,11 +3954,15 @@ void fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_GET_CRC32: + Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 || + (hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED); /* calculate crc32 for a file */ - if (hdr.arg == 1) - crc = pgFileGetCRCgz(buf, true, true); + if ((hdr.arg & GET_CRC32_DECOMPRESS)) + crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); + else if ((hdr.arg & GET_CRC32_TRUNCATED)) + crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); else - crc = pgFileGetCRC(buf, true, true); + crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: @@ -2713,6 +3984,30 @@ void fio_communicate(int in, int out) case FIO_DISCONNECT: hdr.cop = FIO_DISCONNECTED; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + free(buf); + return; + case FIO_GET_ASYNC_ERROR: + fio_get_async_error_impl(out); + break; + case FIO_READLINK: /* Read content of a symbolic link */ + { + /* + * We need a buf for a arguments and for a result at the same time + * hdr.size = strlen(symlink_name) + 1 + * hdr.arg = bufsize for a answer (symlink content) + */ + size_t filename_size = (size_t)hdr.size; + if (filename_size + hdr.arg > buf_size) { + buf_size = hdr.arg; + buf = (char*)realloc(buf, buf_size); + } + rc = readlink(buf, buf + filename_size, hdr.arg); + hdr.cop = FIO_READLINK; + hdr.size = rc > 0 ? rc : 0; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (hdr.size != 0) + IO_CHECK(fio_write_all(out, buf + filename_size, hdr.size), hdr.size); + } break; default: Assert(false); @@ -2724,4 +4019,3 @@ void fio_communicate(int in, int out) exit(EXIT_FAILURE); } } - diff --git a/src/utils/file.h b/src/utils/file.h index 612bda18f..01e5a24f4 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -12,6 +12,8 @@ typedef enum { + /* message for compatibility check */ + FIO_AGENT_VERSION, /* never move this */ FIO_OPEN, FIO_CLOSE, FIO_WRITE, @@ -34,7 +36,7 @@ typedef enum FIO_READDIR, FIO_CLOSEDIR, FIO_PAGE, - FIO_WRITE_COMPRESSED, + FIO_WRITE_COMPRESSED_ASYNC, FIO_GET_CRC32, /* used for incremental restore */ FIO_GET_CHECKSUM_MAP, @@ -50,10 +52,12 @@ typedef enum /* messages for closing connection */ FIO_DISCONNECT, FIO_DISCONNECTED, - /* message for compatibility check */ - FIO_AGENT_VERSION, FIO_LIST_DIR, - FIO_CHECK_POSTMASTER + FIO_CHECK_POSTMASTER, + FIO_GET_ASYNC_ERROR, + FIO_WRITE_ASYNC, + FIO_READLINK, + FIO_PAGE_ZERO } fio_operations; typedef enum @@ -88,10 +92,14 @@ extern fio_location MyLocation; extern void fio_redirect(int in, int out, int err); extern void fio_communicate(int in, int out); -extern int fio_get_agent_version(void); +extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size); extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern ssize_t fio_fwrite_compressed(FILE* f, void const* buf, size_t size, int compress_alg); +extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); +extern size_t fio_fwrite_async(FILE* f, void const* buf, size_t size); +extern int fio_check_error_file(FILE* f, char **errmsg); +extern int fio_check_error_fd(int fd, char **errmsg); +extern int fio_check_error_fd_gz(gzFile f, char **errmsg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); @@ -104,6 +112,7 @@ extern void fio_error(int rc, int size, char const* file, int line); extern int fio_open(char const* name, int mode, fio_location location); extern ssize_t fio_write(int fd, void const* buf, size_t size); +extern ssize_t fio_write_async(int fd, void const* buf, size_t size); extern ssize_t fio_read(int fd, void* buf, size_t size); extern int fio_flush(int fd); extern int fio_seek(int fd, off_t offs); @@ -112,7 +121,10 @@ extern int fio_truncate(int fd, off_t size); extern int fio_close(int fd); extern void fio_disconnect(void); extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); +extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, + bool decompress, bool missing_ok); +extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location, + bool missing_ok); extern int fio_rename(char const* old_path, char const* new_path, fio_location location); extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); @@ -121,6 +133,8 @@ extern int fio_mkdir(char const* path, int mode, fio_location location); extern int fio_chmod(char const* path, int mode, fio_location location); extern int fio_access(char const* path, int mode, fio_location location); extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); +extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location); +extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); extern DIR* fio_opendir(char const* path, fio_location location); extern struct dirent * fio_readdir(DIR *dirp); extern int fio_closedir(DIR *dirp); @@ -138,4 +152,3 @@ extern const char* fio_gzerror(gzFile file, int *errnum); #endif #endif - diff --git a/src/utils/json.c b/src/utils/json.c index 9f13a958f..2c8e0fe9b 100644 --- a/src/utils/json.c +++ b/src/utils/json.c @@ -144,3 +144,21 @@ json_add_escaped(PQExpBuffer buf, const char *str) } appendPQExpBufferChar(buf, '"'); } + +void +json_add_min(PQExpBuffer buf, JsonToken type) +{ + switch (type) + { + case JT_BEGIN_OBJECT: + appendPQExpBufferChar(buf, '{'); + add_comma = false; + break; + case JT_END_OBJECT: + appendPQExpBufferStr(buf, "}\n"); + add_comma = true; + break; + default: + break; + } +} diff --git a/src/utils/json.h b/src/utils/json.h index cc9f1168d..f80832e69 100644 --- a/src/utils/json.h +++ b/src/utils/json.h @@ -25,6 +25,7 @@ typedef enum } JsonToken; extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); +extern void json_add_min(PQExpBuffer buf, JsonToken type); extern void json_add_key(PQExpBuffer buf, const char *name, int32 level); extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, int32 level, bool escaped); diff --git a/src/utils/logger.c b/src/utils/logger.c index 5aee41b46..7ea41f74e 100644 --- a/src/utils/logger.c +++ b/src/utils/logger.c @@ -19,14 +19,19 @@ #include "utils/configuration.h" +#include "json.h" + /* Logger parameters */ LoggerConfig logger_config = { LOG_LEVEL_CONSOLE_DEFAULT, LOG_LEVEL_FILE_DEFAULT, LOG_FILENAME_DEFAULT, NULL, + NULL, LOG_ROTATION_SIZE_DEFAULT, - LOG_ROTATION_AGE_DEFAULT + LOG_ROTATION_AGE_DEFAULT, + LOG_FORMAT_CONSOLE_DEFAULT, + LOG_FORMAT_FILE_DEFAULT }; /* Implementation for logging.h */ @@ -39,6 +44,10 @@ typedef enum PG_FATAL } eLogType; +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING +#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif + void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); static void elog_internal(int elevel, bool file_only, const char *message); @@ -48,7 +57,7 @@ static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf( /* Functions to work with log files */ static void open_logfile(FILE **file, const char *filename_format); -static void release_logfile(void); +static void release_logfile(bool fatal, void *userdata); static char *logfile_getname(const char *format, time_t timestamp); static FILE *logfile_open(const char *filename, const char *mode); @@ -92,6 +101,7 @@ init_logger(const char *root_path, LoggerConfig *config) #if PG_VERSION_NUM >= 120000 /* Setup logging for functions from other modules called by pg_probackup */ pg_logging_init(PROGRAM_NAME); + errno = 0; /* sometimes pg_logging_init sets errno */ switch (logger_config.log_level_console) { @@ -115,6 +125,84 @@ init_logger(const char *root_path, LoggerConfig *config) #endif } +/* + * Check that we are connected to terminal and + * enable ANSI escape codes for Windows if possible + */ +void +init_console(void) +{ + + /* no point in tex coloring if we do not connected to terminal */ + if (!isatty(fileno(stderr)) || + !isatty(fileno(stdout))) + { + show_color = false; + return; + } + +#ifdef WIN32 + HANDLE hOut = INVALID_HANDLE_VALUE; + HANDLE hErr = INVALID_HANDLE_VALUE; + DWORD dwMode_out = 0; + DWORD dwMode_err = 0; + + hOut = GetStdHandle(STD_OUTPUT_HANDLE); + if (hOut == INVALID_HANDLE_VALUE || !hOut) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get terminal stdout handle: %s", strerror(errno)); + return; + } + + hErr = GetStdHandle(STD_ERROR_HANDLE); + if (hErr == INVALID_HANDLE_VALUE || !hErr) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get terminal stderror handle: %s", strerror(errno)); + return; + } + + if (!GetConsoleMode(hOut, &dwMode_out)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get console mode for stdout: %s", strerror(errno)); + return; + } + + if (!GetConsoleMode(hErr, &dwMode_err)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Failed to get console mode for stderr: %s", strerror(errno)); + return; + } + + /* Add ANSI codes support */ + dwMode_out |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + dwMode_err |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; + + if (!SetConsoleMode(hOut, dwMode_out)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Cannot set console mode for stdout: %s", strerror(errno)); + return; + } + + if (!SetConsoleMode(hErr, dwMode_err)) + { + show_color = false; + _dosmaperr(GetLastError()); + elog(WARNING, "Cannot set console mode for stderr: %s", strerror(errno)); + return; + } +#endif +} + static void write_elevel(FILE *stream, int elevel) { @@ -144,6 +232,35 @@ write_elevel(FILE *stream, int elevel) } } +static void +write_elevel_for_json(PQExpBuffer buf, int elevel) +{ + switch (elevel) + { + case VERBOSE: + appendPQExpBufferStr(buf, "\"VERBOSE\""); + break; + case LOG: + appendPQExpBufferStr(buf, "\"LOG\""); + break; + case INFO: + appendPQExpBufferStr(buf, "\"INFO\""); + break; + case NOTICE: + appendPQExpBufferStr(buf, "\"NOTICE\""); + break; + case WARNING: + appendPQExpBufferStr(buf, "\"WARNING\""); + break; + case ERROR: + appendPQExpBufferStr(buf, "\"ERROR\""); + break; + default: + elog_stderr(ERROR, "invalid logging level: %d", elevel); + break; + } +} + /* * Exit with code if it is an error. * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup @@ -193,6 +310,12 @@ elog_internal(int elevel, bool file_only, const char *message) time_t log_time = (time_t) time(NULL); char strfbuf[128]; char str_pid[128]; + char str_pid_json[128]; + char str_thread_json[64]; + PQExpBufferData show_buf; + PQExpBuffer buf_json = &show_buf; + int8 format_console, + format_file; write_to_file = elevel >= logger_config.log_level_file && logger_config.log_directory @@ -200,6 +323,8 @@ elog_internal(int elevel, bool file_only, const char *message) write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && logger_config.log_directory && logger_config.log_directory[0] != '\0'; write_to_stderr = elevel >= logger_config.log_level_console && !file_only; + format_console = logger_config.log_format_console; + format_file = logger_config.log_format_file; if (remote_agent) { @@ -209,10 +334,27 @@ elog_internal(int elevel, bool file_only, const char *message) pthread_lock(&log_file_mutex); loggin_in_progress = true; - if (write_to_file || write_to_error_log || is_archive_cmd) + if (write_to_file || write_to_error_log || is_archive_cmd || + format_console == JSON) strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", localtime(&log_time)); + if (format_file == JSON || format_console == JSON) + { + snprintf(str_pid_json, sizeof(str_pid_json), "%d", my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + + initPQExpBuffer(&show_buf); + json_add_min(buf_json, JT_BEGIN_OBJECT); + json_add_value(buf_json, "ts", strfbuf, 0, true); + json_add_value(buf_json, "pid", str_pid_json, 0, true); + json_add_key(buf_json, "level", 0); + write_elevel_for_json(buf_json, elevel); + json_add_value(buf_json, "msg", message, 0, true); + json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); + json_add_min(buf_json, JT_END_OBJECT); + } + snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); /* @@ -223,18 +365,19 @@ elog_internal(int elevel, bool file_only, const char *message) if (write_to_file) { if (log_file == NULL) + open_logfile(&log_file, logger_config.log_filename ? logger_config.log_filename : LOG_FILENAME_DEFAULT); + if (format_file == JSON) { - if (logger_config.log_filename == NULL) - open_logfile(&log_file, LOG_FILENAME_DEFAULT); - else - open_logfile(&log_file, logger_config.log_filename); + fputs(buf_json->data, log_file); } + else + { + fprintf(log_file, "%s ", strfbuf); + fprintf(log_file, "%s ", str_pid); + write_elevel(log_file, elevel); - fprintf(log_file, "%s ", strfbuf); - fprintf(log_file, "%s ", str_pid); - write_elevel(log_file, elevel); - - fprintf(log_file, "%s\n", message); + fprintf(log_file, "%s\n", message); + } fflush(log_file); } @@ -248,11 +391,18 @@ elog_internal(int elevel, bool file_only, const char *message) if (error_log_file == NULL) open_logfile(&error_log_file, logger_config.error_log_filename); - fprintf(error_log_file, "%s ", strfbuf); - fprintf(error_log_file, "%s ", str_pid); - write_elevel(error_log_file, elevel); + if (format_file == JSON) + { + fputs(buf_json->data, error_log_file); + } + else + { + fprintf(error_log_file, "%s ", strfbuf); + fprintf(error_log_file, "%s ", str_pid); + write_elevel(error_log_file, elevel); - fprintf(error_log_file, "%s\n", message); + fprintf(error_log_file, "%s\n", message); + } fflush(error_log_file); } @@ -262,20 +412,48 @@ elog_internal(int elevel, bool file_only, const char *message) */ if (write_to_stderr) { - if (is_archive_cmd) + if (format_console == JSON) + { + fprintf(stderr, "%s", buf_json->data); + } + else { - char str_thread[64]; - /* [Issue #213] fix pgbadger parsing */ - snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); + if (is_archive_cmd) + { + char str_thread[64]; + /* [Issue #213] fix pgbadger parsing */ + snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); - fprintf(stderr, "%s ", strfbuf); - fprintf(stderr, "%s ", str_pid); - fprintf(stderr, "%s ", str_thread); + fprintf(stderr, "%s ", strfbuf); + fprintf(stderr, "%s ", str_pid); + fprintf(stderr, "%s ", str_thread); + } + else if (show_color) + { + /* color WARNING and ERROR messages */ + if (elevel == WARNING) + fprintf(stderr, "%s", TC_YELLOW_BOLD); + else if (elevel == ERROR) + fprintf(stderr, "%s", TC_RED_BOLD); + } + + write_elevel(stderr, elevel); + + /* main payload */ + fprintf(stderr, "%s", message); + + /* reset color to default */ + if (show_color && (elevel == WARNING || elevel == ERROR)) + fprintf(stderr, "%s", TC_RESET); + + fprintf(stderr, "\n"); } - write_elevel(stderr, elevel); + if (format_file == JSON || format_console == JSON) + { + termPQExpBuffer(buf_json); + } - fprintf(stderr, "%s\n", message); fflush(stderr); } @@ -292,7 +470,15 @@ elog_internal(int elevel, bool file_only, const char *message) static void elog_stderr(int elevel, const char *fmt, ...) { - va_list args; + va_list args; + PQExpBufferData show_buf; + PQExpBuffer buf_json = &show_buf; + time_t log_time = (time_t) time(NULL); + char strfbuf[128]; + char str_pid[128]; + char str_thread_json[64]; + char *message; + int8 format_console; /* * Do not log message if severity level is less than log_level. @@ -303,11 +489,37 @@ elog_stderr(int elevel, const char *fmt, ...) va_start(args, fmt); - write_elevel(stderr, elevel); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - fflush(stderr); + format_console = logger_config.log_format_console; + if (format_console == JSON) + { + strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", + localtime(&log_time)); + snprintf(str_pid, sizeof(str_pid), "%d", my_pid); + snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); + + initPQExpBuffer(&show_buf); + json_add_min(buf_json, JT_BEGIN_OBJECT); + json_add_value(buf_json, "ts", strfbuf, 0, true); + json_add_value(buf_json, "pid", str_pid, 0, true); + json_add_key(buf_json, "level", 0); + write_elevel_for_json(buf_json, elevel); + message = get_log_message(fmt, args); + json_add_value(buf_json, "msg", message, 0, true); + json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); + json_add_min(buf_json, JT_END_OBJECT); + fputs(buf_json->data, stderr); + pfree(message); + termPQExpBuffer(buf_json); + } + else + { + write_elevel(stderr, elevel); + vfprintf(stderr, fmt, args); + fputc('\n', stderr); + } + + fflush(stderr); va_end(args); exit_if_necessary(elevel); @@ -476,6 +688,36 @@ parse_log_level(const char *level) return 0; } +int +parse_log_format(const char *format) +{ + const char *v = format; + size_t len; + + if (v == NULL) + { + elog(ERROR, "log-format got invalid value"); + return 0; + } + + /* Skip all spaces detected */ + while (isspace((unsigned char)*v)) + v++; + len = strlen(v); + + if (len == 0) + elog(ERROR, "log-format is empty"); + + if (pg_strncasecmp("plain", v, len) == 0) + return PLAIN; + else if (pg_strncasecmp("json", v, len) == 0) + return JSON; + + /* Log format is invalid */ + elog(ERROR, "invalid log-format \"%s\"", format); + return 0; +} + /* * Converts integer representation of log level to string. */ @@ -505,6 +747,22 @@ deparse_log_level(int level) return NULL; } +const char * +deparse_log_format(int format) +{ + switch (format) + { + case PLAIN: + return "PLAIN"; + case JSON: + return "JSON"; + default: + elog(ERROR, "invalid log-format %d", format); + } + + return NULL; +} + /* * Construct logfile name using timestamp information. * @@ -698,7 +956,7 @@ open_logfile(FILE **file, const char *filename_format) */ if (!exit_hook_registered) { - atexit(release_logfile); + pgut_atexit_push(release_logfile, NULL); exit_hook_registered = true; } } @@ -707,7 +965,7 @@ open_logfile(FILE **file, const char *filename_format) * Closes opened file. */ static void -release_logfile(void) +release_logfile(bool fatal, void *userdata) { if (log_file) { diff --git a/src/utils/logger.h b/src/utils/logger.h index 37b6ff095..adc5061e0 100644 --- a/src/utils/logger.h +++ b/src/utils/logger.h @@ -21,6 +21,9 @@ #define ERROR 1 #define LOG_OFF 10 +#define PLAIN 0 +#define JSON 1 + typedef struct LoggerConfig { int log_level_console; @@ -32,6 +35,8 @@ typedef struct LoggerConfig uint64 log_rotation_size; /* Maximum lifetime of an individual log file in minutes */ uint64 log_rotation_age; + int8 log_format_console; + int8 log_format_file; } LoggerConfig; /* Logger parameters */ @@ -43,6 +48,9 @@ extern LoggerConfig logger_config; #define LOG_LEVEL_CONSOLE_DEFAULT INFO #define LOG_LEVEL_FILE_DEFAULT LOG_OFF +#define LOG_FORMAT_CONSOLE_DEFAULT PLAIN +#define LOG_FORMAT_FILE_DEFAULT PLAIN + #define LOG_FILENAME_DEFAULT "pg_probackup.log" #define LOG_DIRECTORY_DEFAULT "log" @@ -51,8 +59,11 @@ extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); extern void init_logger(const char *root_path, LoggerConfig *config); +extern void init_console(void); extern int parse_log_level(const char *level); extern const char *deparse_log_level(int level); +extern int parse_log_format(const char *format); +extern const char *deparse_log_format(int format); #endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c index 31148ee9a..65377c001 100644 --- a/src/utils/parray.c +++ b/src/utils/parray.c @@ -175,7 +175,7 @@ parray_rm(parray *array, const void *key, int(*compare)(const void *, const void size_t parray_num(const parray *array) { - return array->used; + return array!= NULL ? array->used : (size_t) 0; } void @@ -198,6 +198,13 @@ parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const return bsearch(&key, array->data, array->used, sizeof(void *), compare); } +int +parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)) +{ + void **elem = parray_bsearch(array, key, compare); + return elem != NULL ? elem - array->data : -1; +} + /* checks that parray contains element */ bool parray_contains(parray *array, void *elem) { @@ -210,3 +217,30 @@ bool parray_contains(parray *array, void *elem) } return false; } + +/* effectively remove elements that satisfy certain criterion */ +void +parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean) { + int i = 0; + int j = 0; + + /* removing certain elements */ + while(j < parray_num(array)) { + void *value = array->data[j]; + // if the value satisfies the criterion, clean it up + if(criterion(value, args)) { + clean(value); + j++; + continue; + } + + if(i != j) + array->data[i] = array->data[j]; + + i++; + j++; + } + + /* adjust the number of used elements */ + array->used -= j - i; +} diff --git a/src/utils/parray.h b/src/utils/parray.h index 85d7383f3..08846f252 100644 --- a/src/utils/parray.h +++ b/src/utils/parray.h @@ -16,6 +16,9 @@ */ typedef struct parray parray; +typedef bool (*criterion_fn)(void *value, void *args); +typedef void (*cleanup_fn)(void *ref); + extern parray *parray_new(void); extern void parray_expand(parray *array, size_t newnum); extern void parray_free(parray *array); @@ -29,8 +32,10 @@ extern bool parray_rm(parray *array, const void *key, int(*compare)(const void * extern size_t parray_num(const parray *array); extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); +extern int parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)); extern void parray_walk(parray *array, void (*action)(void *)); extern bool parray_contains(parray *array, void *elem); +extern void parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean); #endif /* PARRAY_H */ diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 6d996f47f..9559fa644 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -3,7 +3,7 @@ * pgut.c * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2019, Postgres Professional + * Portions Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -16,6 +16,16 @@ #include "libpq/pqsignal.h" #include "pqexpbuffer.h" +#if PG_VERSION_NUM >= 140000 +#include "common/string.h" +#endif + +#if PG_VERSION_NUM >= 100000 +#include "common/connect.h" +#else +#include "fe_utils/connect.h" +#endif + #include #include "pgut.h" @@ -35,6 +45,9 @@ bool interrupted = false; bool in_cleanup = false; bool in_password = false; +/* critical section when adding disconnect callbackups */ +static pthread_mutex_t atexit_callback_disconnect_mutex = PTHREAD_MUTEX_INITIALIZER; + /* Connection routines */ static void init_cancel_handler(void); static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); @@ -48,6 +61,7 @@ static void pgut_pgfnames_cleanup(char **filenames); void discard_response(PGconn *conn); +/* Note that atexit handlers always called on the main thread */ void pgut_init(void) { @@ -71,7 +85,16 @@ prompt_for_password(const char *username) password = NULL; } -#if PG_VERSION_NUM >= 100000 +#if PG_VERSION_NUM >= 140000 + if (username == NULL) + password = simple_prompt("Password: ", false); + else + { + char message[256]; + snprintf(message, lengthof(message), "Password for user %s: ", username); + password = simple_prompt(message , false); + } +#elif PG_VERSION_NUM >= 100000 password = (char *) pgut_malloc(sizeof(char) * 100 + 1); if (username == NULL) simple_prompt("Password: ", password, 100, false); @@ -237,8 +260,10 @@ pgut_connect(const char *host, const char *port, if (PQstatus(conn) == CONNECTION_OK) { + pthread_lock(&atexit_callback_disconnect_mutex); pgut_atexit_push(pgut_disconnect_callback, conn); - return conn; + pthread_mutex_unlock(&atexit_callback_disconnect_mutex); + break; } if (conn && PQconnectionNeedsPassword(conn) && prompt_password) @@ -260,11 +285,34 @@ pgut_connect(const char *host, const char *port, PQfinish(conn); return NULL; } + + /* + * Fix for CVE-2018-1058. This code was taken with small modification from + * src/bin/pg_basebackup/streamutil.c:GetConnection() + */ + if (dbname != NULL) + { + PGresult *res; + + res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + elog(ERROR, "could not clear search_path: %s", + PQerrorMessage(conn)); + PQclear(res); + PQfinish(conn); + return NULL; + } + PQclear(res); + } + + return conn; } PGconn * pgut_connect_replication(const char *host, const char *port, - const char *dbname, const char *username) + const char *dbname, const char *username, + bool strict) { PGconn *tmpconn; int argcount = 7; /* dbname, replication, fallback_app_name, @@ -350,7 +398,7 @@ pgut_connect_replication(const char *host, const char *port, continue; } - elog(ERROR, "could not connect to database %s: %s", + elog(strict ? ERROR : WARNING, "could not connect to database %s: %s", dbname, PQerrorMessage(tmpconn)); PQfinish(tmpconn); free(values); @@ -365,7 +413,10 @@ pgut_disconnect(PGconn *conn) { if (conn) PQfinish(conn); + + pthread_lock(&atexit_callback_disconnect_mutex); pgut_atexit_pop(pgut_disconnect_callback, conn); + pthread_mutex_unlock(&atexit_callback_disconnect_mutex); } @@ -840,7 +891,9 @@ call_atexit_callbacks(bool fatal) { pgut_atexit_item *item; pgut_atexit_item *next; - for (item = pgut_atexit_stack; item; item = next){ + + for (item = pgut_atexit_stack; item; item = next) + { next = item->next; item->callback(fatal, item->userdata); } @@ -865,6 +918,17 @@ pgut_malloc(size_t size) return ret; } +void * +pgut_malloc0(size_t size) +{ + char *ret; + + ret = pgut_malloc(size); + memset(ret, 0, size); + + return ret; +} + void * pgut_realloc(void *p, size_t size) { @@ -890,6 +954,51 @@ pgut_strdup(const char *str) return ret; } +char * +pgut_strndup(const char *str, size_t n) +{ + char *ret; + + if (str == NULL) + return NULL; + +#if _POSIX_C_SOURCE >= 200809L + if ((ret = strndup(str, n)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); +#else /* WINDOWS doesn't have strndup() */ + if ((ret = malloc(n + 1)) == NULL) + elog(ERROR, "could not duplicate string \"%s\": %s", + str, strerror(errno)); + + memcpy(ret, str, n); + ret[n] = '\0'; +#endif + return ret; +} + +/* + * Allocates new string, that contains part of filepath string minus trailing filename string + * If trailing filename string not found, returns copy of filepath. + * Result must be free by caller. + */ +char * +pgut_str_strip_trailing_filename(const char *filepath, const char *filename) +{ + size_t fp_len = strlen(filepath); + size_t fn_len = strlen(filename); + if (strncmp(filepath + fp_len - fn_len, filename, fn_len) == 0) + return pgut_strndup(filepath, fp_len - fn_len); + else + return pgut_strndup(filepath, fp_len); +} + +void +pgut_free(void *p) +{ + free(p); +} + FILE * pgut_fopen(const char *path, const char *mode, bool missing_ok) { @@ -956,6 +1065,7 @@ init_cancel_handler(void) oldhandler = pqsignal(SIGINT, handle_interrupt); pqsignal(SIGQUIT, handle_interrupt); pqsignal(SIGTERM, handle_interrupt); + pqsignal(SIGPIPE, handle_interrupt); } #else /* WIN32 */ @@ -1105,13 +1215,16 @@ pgut_pgfnames(const char *path, bool strict) } } + filenames[numnames] = NULL; + if (errno) { elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path); + pgut_pgfnames_cleanup(filenames); + closedir(dir); return NULL; } - filenames[numnames] = NULL; if (closedir(dir)) { @@ -1160,7 +1273,7 @@ pgut_rmtree(const char *path, bool rmtopdir, bool strict) /* now we have the names we can start removing things */ for (filename = filenames; *filename; filename++) { - snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename); + join_path_components(pathbuf, path, *filename); if (lstat(pathbuf, &statbuf) != 0) { @@ -1202,3 +1315,61 @@ pgut_rmtree(const char *path, bool rmtopdir, bool strict) return result; } + +/* cross-platform setenv */ +void +pgut_setenv(const char *key, const char *val) +{ +#ifdef WIN32 + char *envstr = NULL; + envstr = psprintf("%s=%s", key, val); + putenv(envstr); +#else + setenv(key, val, 1); +#endif +} + +/* stolen from unsetenv.c */ +void +pgut_unsetenv(const char *key) +{ +#ifdef WIN32 + char *envstr = NULL; + + if (getenv(key) == NULL) + return; /* no work */ + + /* + * The technique embodied here works if libc follows the Single Unix Spec + * and actually uses the storage passed to putenv() to hold the environ + * entry. When we clobber the entry in the second step we are ensuring + * that we zap the actual environ member. However, there are some libc + * implementations (notably recent BSDs) that do not obey SUS but copy the + * presented string. This method fails on such platforms. Hopefully all + * such platforms have unsetenv() and thus won't be using this hack. See: + * http://www.greenend.org.uk/rjk/2008/putenv.html + * + * Note that repeatedly setting and unsetting a var using this code will + * leak memory. + */ + + envstr = (char *) pgut_malloc(strlen(key) + 2); + if (!envstr) /* not much we can do if no memory */ + return; + + /* Override the existing setting by forcibly defining the var */ + sprintf(envstr, "%s=", key); + putenv(envstr); + + /* Now we can clobber the variable definition this way: */ + strcpy(envstr, "="); + + /* + * This last putenv cleans up if we have multiple zero-length names as a + * result of unsetting multiple things. + */ + putenv(envstr); +#else + unsetenv(key); +#endif +} diff --git a/src/utils/pgut.h b/src/utils/pgut.h index d196aad3d..1b7b7864c 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2019, Postgres Professional + * Portions Copyright (c) 2017-2021, Postgres Professional * *------------------------------------------------------------------------- */ @@ -40,8 +40,8 @@ extern char *pgut_get_conninfo_string(PGconn *conn); extern PGconn *pgut_connect(const char *host, const char *port, const char *dbname, const char *username); extern PGconn *pgut_connect_replication(const char *host, const char *port, - const char *dbname, - const char *username); + const char *dbname, const char *username, + bool strict); extern void pgut_disconnect(PGconn *conn); extern void pgut_disconnect_callback(bool fatal, void *userdata); extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, @@ -59,10 +59,15 @@ extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); * memory allocators */ extern void *pgut_malloc(size_t size); +extern void *pgut_malloc0(size_t size); extern void *pgut_realloc(void *p, size_t size); extern char *pgut_strdup(const char *str); +extern char *pgut_strndup(const char *str, size_t n); +extern char *pgut_str_strip_trailing_filename(const char *filepath, const char *filename); +extern void pgut_free(void *p); #define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) +#define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) /* @@ -104,4 +109,20 @@ extern int sleep(unsigned int seconds); extern int usleep(unsigned int usec); #endif +#ifdef _MSC_VER +#define ARG_SIZE_HINT +#else +#define ARG_SIZE_HINT static +#endif + +static inline uint32 hash_mix32_2(uint32 a, uint32 b) +{ + b ^= (a<<7)|(a>>25); + a *= 0xdeadbeef; + b *= 0xcafeabed; + a ^= a >> 16; + b ^= b >> 15; + return a^b; +} + #endif /* PGUT_H */ diff --git a/src/utils/remote.c b/src/utils/remote.c index f590a82b4..7ef8d3239 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -82,6 +82,11 @@ void wait_ssh(void) #endif } +/* + * On windows we launch a new pbk process via 'pg_probackup ssh ...' + * so this process would new that it should exec ssh, because + * there is no fork on Windows. + */ #ifdef WIN32 void launch_ssh(char* argv[]) { @@ -142,6 +147,9 @@ bool launch_agent(void) ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "Compression=no"; + ssh_argv[ssh_argc++] = "-o"; + ssh_argv[ssh_argc++] = "ControlMaster=no"; + ssh_argv[ssh_argc++] = "-o"; ssh_argv[ssh_argc++] = "LogLevel=error"; @@ -221,7 +229,7 @@ bool launch_agent(void) return false; } else { #endif - elog(LOG, "Start SSH client process, pid %d", child_pid); + elog(LOG, "Start SSH client process, pid %d, cmd \"%s\"", child_pid, cmd); SYS_CHECK(close(infd[1])); /* These are being used by the child */ SYS_CHECK(close(outfd[0])); SYS_CHECK(close(errfd[1])); @@ -230,10 +238,114 @@ bool launch_agent(void) fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ } - /* Make sure that remote agent has the same version - * TODO: we must also check PG version and fork edition - */ - agent_version = fio_get_agent_version(); + + /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ + { + char payload_buf[1024]; + fio_get_agent_version(&agent_version, payload_buf, sizeof payload_buf); + check_remote_agent_compatibility(agent_version, payload_buf, sizeof payload_buf); + } + + return true; +} + +#ifdef PGPRO_EDITION +/* PGPRO 10-13 checks to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ +static bool check_certified() +{ + return strstr(PGPRO_VERSION_STR, "(certified)") || + strstr(PGPRO_VERSION_STR, "(standard certified)"); +} +#endif + +static char* extract_pg_edition_str() +{ + static char *vanilla = "vanilla"; +#ifdef PGPRO_EDITION + static char *_1C = "1C"; + static char *std = "standard"; + static char *ent = "enterprise"; + static char *std_cert = "standard-certified"; + static char *ent_cert = "enterprise-certified"; + + if (strcmp(PGPRO_EDITION, _1C) == 0) + return vanilla; + + if (PG_VERSION_NUM < 100000) + return PGPRO_EDITION; + + /* these "certified" checks are applicable to PGPRO from 10 up to 12 versions. + * 13+ certified versions are compatible to non-certified ones */ + if (PG_VERSION_NUM < 130000 && check_certified()) + { + if (strcmp(PGPRO_EDITION, std) == 0) + return std_cert; + else if (strcmp(PGPRO_EDITION, ent) == 0) + return ent_cert; + else + Assert("Bad #define PGPRO_EDITION value" == 0); + } + + return PGPRO_EDITION; +#else + return vanilla; +#endif +} + +#define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } +#define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } + +#define COMPATIBILITY_VAL_SEPARATOR "=" +#define COMPATIBILITY_LINE_SEPARATOR "\n" + +/* + * Compose compatibility string to be sent by pg_probackup agent + * through ssh and to be verified by pg_probackup peer. + * Compatibility string contains postgres essential vars as strings + * in format "var_name" + COMPATIBILITY_VAL_SEPARATOR + "var_value" + COMPATIBILITY_LINE_SEPARATOR + */ +size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) +{ + typedef struct compatibility_param_tag { + const char* name; + const char* strval; + int intval; + } compatibility_param; + + compatibility_param compatibility_params[] = { + COMPATIBILITY_VAL_STR(PG_MAJORVERSION), + { "edition", extract_pg_edition_str(), 0 }, + COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), + }; + + size_t result_size = 0; + int i; + *compatibility_buf = '\0'; + + for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) + { + if (compatibility_params[i].strval != NULL) + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i].name, + compatibility_params[i].strval); + else + result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, + "%s" COMPATIBILITY_VAL_SEPARATOR "%d" COMPATIBILITY_LINE_SEPARATOR, + compatibility_params[i].name, + compatibility_params[i].intval); + Assert(result_size < compatibility_buf_size); + } + return result_size + 1; +} + +/* + * Check incoming remote agent's compatibility params for equality to local ones. + */ +void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) +{ + elog(LOG, "Agent version=%d\n", agent_version); + if (agent_version != AGENT_PROTOCOL_VERSION) { char agent_version_str[1024]; @@ -242,9 +354,26 @@ bool launch_agent(void) (agent_version / 100) % 100, agent_version % 100); - elog(ERROR, "Remote agent version %s does not match local program version %s", - agent_version_str, PROGRAM_VERSION); + elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " + "consider to upgrade pg_probackup binary", + agent_version_str, AGENT_PROTOCOL_VERSION_STR); } - return true; + /* checking compatibility params */ + if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) + { + elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); + } + + elog(LOG, "Agent compatibility params:\n%s", compatibility_str); + + { + char buf[1024]; + + prepare_compatibility_str(buf, sizeof buf); + if(strcmp(compatibility_str, buf)) + { + elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s", buf, compatibility_str); + } + } } diff --git a/src/utils/thread.c b/src/utils/thread.c index 5ceee068d..1c469bd29 100644 --- a/src/utils/thread.c +++ b/src/utils/thread.c @@ -11,6 +11,10 @@ #include "thread.h" +/* + * Global var used to detect error condition (not signal interrupt!) in threads, + * so if one thread errored out, then others may abort + */ bool thread_interrupted = false; #ifdef WIN32 diff --git a/src/validate.c b/src/validate.c index d16d27677..0887b2e7a 100644 --- a/src/validate.c +++ b/src/validate.c @@ -16,7 +16,7 @@ #include "utils/thread.h" static void *pgBackupValidateFiles(void *arg); -static void do_validate_instance(void); +static void do_validate_instance(InstanceState *instanceState); static bool corrupted_backup_found = false; static bool skipped_due_to_lock = false; @@ -48,7 +48,6 @@ typedef struct void pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { - char base_path[MAXPGPATH]; char external_prefix[MAXPGPATH]; parray *files = NULL; bool corrupted = false; @@ -64,19 +63,19 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " "pg_probackup do not guarantee to be forward compatible. " "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, base36enc(backup->start_time), backup->program_version); + PROGRAM_VERSION, backup_id_of(backup), backup->program_version); /* Check backup server version */ if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " "compiled with server version %s", - base36enc(backup->start_time), backup->server_version, PG_MAJORVERSION); + backup_id_of(backup), backup->server_version, PG_MAJORVERSION); if (backup->status == BACKUP_STATUS_RUNNING) { elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", - base36enc(backup->start_time), status2str(backup->status)); - write_backup_status(backup, BACKUP_STATUS_ERROR, instance_name, true); + backup_id_of(backup), status2str(backup->status)); + write_backup_status(backup, BACKUP_STATUS_ERROR, true); corrupted_backup_found = true; return; } @@ -89,7 +88,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status != BACKUP_STATUS_CORRUPT) { elog(WARNING, "Backup %s has status %s. Skip validation.", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); corrupted_backup_found = true; return; } @@ -99,31 +98,30 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) backup->status == BACKUP_STATUS_MERGING) { elog(WARNING, "Full backup %s has status %s, skip validation", - base36enc(backup->start_time), status2str(backup->status)); + backup_id_of(backup), status2str(backup->status)); return; } if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE || backup->status == BACKUP_STATUS_MERGING) - elog(INFO, "Validating backup %s", base36enc(backup->start_time)); + elog(INFO, "Validating backup %s", backup_id_of(backup)); else - elog(INFO, "Revalidating backup %s", base36enc(backup->start_time)); + elog(INFO, "Revalidating backup %s", backup_id_of(backup)); if (backup->backup_mode != BACKUP_MODE_FULL && backup->backup_mode != BACKUP_MODE_DIFF_PAGE && backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && backup->backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(WARNING, "Invalid backup_mode of backup %s", base36enc(backup->start_time)); + elog(WARNING, "Invalid backup_mode of backup %s", backup_id_of(backup)); - join_path_components(base_path, backup->root_dir, DATABASE_DIR); join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); files = get_backup_filelist(backup, false); if (!files) { - elog(WARNING, "Backup %s file list is corrupted", base36enc(backup->start_time)); + elog(WARNING, "Backup %s file list is corrupted", backup_id_of(backup)); backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); return; } @@ -132,11 +130,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) // params->partial_restore_type); /* setup threads */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - pg_atomic_clear_flag(&file->lock); - } + pfilearray_clear_locks(files); /* init thread args with own file lists */ threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); @@ -149,7 +143,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { validate_files_arg *arg = &(threads_args[i]); - arg->base_path = base_path; + arg->base_path = backup->database_dir; arg->files = files; arg->corrupted = false; arg->backup_mode = backup->backup_mode; @@ -190,13 +184,14 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) /* Update backup status */ if (corrupted) backup->status = BACKUP_STATUS_CORRUPT; + write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK, instance_name, true); + BACKUP_STATUS_OK, true); if (corrupted) - elog(WARNING, "Backup %s data files are corrupted", base36enc(backup->start_time)); + elog(WARNING, "Backup %s data files are corrupted", backup_id_of(backup)); else - elog(INFO, "Backup %s data files are valid", base36enc(backup->start_time)); + elog(INFO, "Backup %s data files are valid", backup_id_of(backup)); /* Issue #132 kludge */ if (!corrupted && @@ -206,7 +201,6 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) { char path[MAXPGPATH]; - //pgBackupGetPath(backup, path, lengthof(path), DATABASE_FILE_LIST); join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); if (pgFileSize(path) >= (BLCKSZ*500)) @@ -214,11 +208,10 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) elog(WARNING, "Backup %s is a victim of metadata corruption. " "Additional information can be found here: " "https://github.com/postgrespro/pg_probackup/issues/132", - base36enc(backup->start_time)); + backup_id_of(backup)); backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); } - } } @@ -383,30 +376,29 @@ pgBackupValidateFiles(void *arg) /* * Validate all backups in the backup catalog. * If --instance option was provided, validate only backups of this instance. + * + * TODO: split into two functions: do_validate_catalog and do_validate_instance. */ int -do_validate_all(void) +do_validate_all(CatalogState *catalogState, InstanceState *instanceState) { corrupted_backup_found = false; skipped_due_to_lock = false; - if (instance_name == NULL) + if (instanceState == NULL) { /* Show list of instances */ - char path[MAXPGPATH]; DIR *dir; struct dirent *dent; /* open directory and list contents */ - join_path_components(path, backup_path, BACKUPS_DIR); - dir = opendir(path); + dir = opendir(catalogState->backup_subdir_path); if (dir == NULL) - elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno)); + elog(ERROR, "Cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); errno = 0; while ((dent = readdir(dir))) { - char conf_path[MAXPGPATH]; char child[MAXPGPATH]; struct stat st; @@ -415,10 +407,10 @@ do_validate_all(void) strcmp(dent->d_name, "..") == 0) continue; - join_path_components(child, path, dent->d_name); + join_path_components(child, catalogState->backup_subdir_path, dent->d_name); if (lstat(child, &st) == -1) - elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno)); + elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno)); if (!S_ISDIR(st.st_mode)) continue; @@ -426,26 +418,30 @@ do_validate_all(void) /* * Initialize instance configuration. */ - instance_name = dent->d_name; - sprintf(backup_instance_path, "%s/%s/%s", - backup_path, BACKUPS_DIR, instance_name); - sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance_name); - join_path_components(conf_path, backup_instance_path, - BACKUP_CATALOG_CONF_FILE); - if (config_read_opt(conf_path, instance_options, ERROR, false, + instanceState = pgut_new(InstanceState); /* memory leak */ + strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); + + join_path_components(instanceState->instance_backup_subdir_path, + catalogState->backup_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_wal_subdir_path, + catalogState->wal_subdir_path, instanceState->instance_name); + join_path_components(instanceState->instance_config_path, + instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); + + if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false, true) == 0) { - elog(WARNING, "Configuration file \"%s\" is empty", conf_path); + elog(WARNING, "Configuration file \"%s\" is empty", instanceState->instance_config_path); corrupted_backup_found = true; continue; } - do_validate_instance(); + do_validate_instance(instanceState); } } else { - do_validate_instance(); + do_validate_instance(instanceState); } /* TODO: Probably we should have different exit code for every condition @@ -475,17 +471,17 @@ do_validate_all(void) * Validate all backups in the given instance of the backup catalog. */ static void -do_validate_instance(void) +do_validate_instance(InstanceState *instanceState) { int i; int j; parray *backups; pgBackup *current_backup = NULL; - elog(INFO, "Validate backups of the instance '%s'", instance_name); + elog(INFO, "Validate backups of the instance '%s'", instanceState->instance_name); /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instance_name, INVALID_BACKUP_ID); + backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); /* Examine backups one by one and validate them */ for (i = 0; i < parray_num(backups); i++) @@ -505,27 +501,27 @@ do_validate_instance(void) /* chain is broken */ if (result == ChainIsBroken) { - char *parent_backup_id; + const char *parent_backup_id; + const char *current_backup_id; /* determine missing backup ID */ - parent_backup_id = base36enc_dup(tmp_backup->parent_backup); + parent_backup_id = base36enc(tmp_backup->parent_backup); + current_backup_id = backup_id_of(current_backup); corrupted_backup_found = true; /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - base36enc(current_backup->start_time), - parent_backup_id); + current_backup_id, parent_backup_id); } else { elog(WARNING, "Backup %s has missing parent %s", - base36enc(current_backup->start_time), parent_backup_id); + current_backup_id, parent_backup_id); } - pg_free(parent_backup_id); continue; } /* chain is whole, but at least one parent is invalid */ @@ -534,23 +530,23 @@ do_validate_instance(void) /* Oldest corrupt backup has a chance for revalidation */ if (current_backup->start_time != tmp_backup->start_time) { - char *backup_id = base36enc_dup(tmp_backup->start_time); /* orphanize current_backup */ if (current_backup->status == BACKUP_STATUS_OK || current_backup->status == BACKUP_STATUS_DONE) { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(current_backup->start_time), backup_id, + backup_id_of(current_backup), + backup_id_of(tmp_backup), status2str(tmp_backup->status)); } else { elog(WARNING, "Backup %s has parent %s with status: %s", - base36enc(current_backup->start_time), backup_id, + backup_id_of(current_backup), + backup_id_of(tmp_backup), status2str(tmp_backup->status)); } - pg_free(backup_id); continue; } base_full_backup = find_parent_full_backup(current_backup); @@ -558,7 +554,7 @@ do_validate_instance(void) /* sanity */ if (!base_full_backup) elog(ERROR, "Parent full backup for the given backup %s was not found", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); } /* chain is whole, all parents are valid at first glance, * current backup validation can proceed @@ -570,10 +566,10 @@ do_validate_instance(void) base_full_backup = current_backup; /* Do not interrupt, validate the next backup */ - if (!lock_backup(current_backup, true)) + if (!lock_backup(current_backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(current_backup->start_time)); + backup_id_of(current_backup)); skipped_due_to_lock = true; continue; } @@ -582,8 +578,8 @@ do_validate_instance(void) /* Validate corresponding WAL files */ if (current_backup->status == BACKUP_STATUS_OK) - validate_wal(current_backup, arclog_path, 0, - 0, 0, base_full_backup->tli, + validate_wal(current_backup, instanceState->instance_wal_subdir_path, 0, + 0, 0, current_backup->tli, instance_config.xlog_seg_size); /* @@ -591,7 +587,6 @@ do_validate_instance(void) */ if (current_backup->status != BACKUP_STATUS_OK) { - char *current_backup_id; /* This is ridiculous but legal. * PAGE_b2 <- OK * PAGE_a2 <- OK @@ -601,7 +596,6 @@ do_validate_instance(void) */ corrupted_backup_found = true; - current_backup_id = base36enc_dup(current_backup->start_time); for (j = i - 1; j >= 0; j--) { @@ -612,16 +606,15 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE) { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, instance_name, true); + write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - base36enc(backup->start_time), - current_backup_id, + backup_id_of(backup), + backup_id_of(current_backup), status2str(current_backup->status)); } } } - free(current_backup_id); } /* For every OK backup we try to revalidate all his ORPHAN descendants. */ @@ -665,10 +658,10 @@ do_validate_instance(void) if (backup->status == BACKUP_STATUS_ORPHAN) { /* Do not interrupt, validate the next backup */ - if (!lock_backup(backup, true)) + if (!lock_backup(backup, true, false)) { elog(WARNING, "Cannot lock backup %s directory, skip validation", - base36enc(backup->start_time)); + backup_id_of(backup)); skipped_due_to_lock = true; continue; } @@ -679,8 +672,8 @@ do_validate_instance(void) { /* Revalidation successful, validate corresponding WAL files */ - validate_wal(backup, arclog_path, 0, - 0, 0, current_backup->tli, + validate_wal(backup, instanceState->instance_wal_subdir_path, 0, + 0, 0, backup->tli, instance_config.xlog_seg_size); } } @@ -700,3 +693,60 @@ do_validate_instance(void) parray_walk(backups, pgBackupFree); parray_free(backups); } + +/* + * Validate tablespace_map checksum. + * Error out in case of checksum mismatch. + * Return 'false' if there are no tablespaces in backup. + * + * TODO: it is a bad, that we read the whole filelist just for + * the sake of tablespace_map. Probably pgBackup should come with + * already filled pgBackup.files + */ +bool +validate_tablespace_map(pgBackup *backup, bool no_validate) +{ + char map_path[MAXPGPATH]; + pgFile *dummy = NULL; + pgFile **tablespace_map = NULL; + pg_crc32 crc; + parray *files = get_backup_filelist(backup, true); + bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || + parse_program_version(backup->program_version) >= 20025; + + parray_qsort(files, pgFileCompareRelPathWithExternal); + join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); + + dummy = pgFileInit(PG_TABLESPACE_MAP_FILE); + tablespace_map = (pgFile **) parray_bsearch(files, dummy, pgFileCompareRelPathWithExternal); + + if (!tablespace_map) + { + elog(LOG, "there is no file tablespace_map"); + parray_walk(files, pgFileFree); + parray_free(files); + return false; + } + + /* Exit if database/tablespace_map doesn't exist */ + if (!fileExists(map_path, FIO_BACKUP_HOST)) + elog(ERROR, "Tablespace map is missing: \"%s\", " + "probably backup %s is corrupt, validate it", + map_path, backup_id_of(backup)); + + /* check tablespace map checksumms */ + if (!no_validate) + { + crc = pgFileGetCRC(map_path, use_crc32c, false); + + if ((*tablespace_map)->crc != crc) + elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " + "probably backup %s is corrupt, validate it", + map_path, crc, (*tablespace_map)->crc, backup_id_of(backup)); + } + + pgFileFree(dummy); + parray_walk(files, pgFileFree); + parray_free(files); + return true; +} diff --git a/tests/CVE_2018_1058_test.py b/tests/CVE_2018_1058_test.py new file mode 100644 index 000000000..cfd55cc60 --- /dev/null +++ b/tests/CVE_2018_1058_test.py @@ -0,0 +1,129 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +class CVE_2018_1058(ProbackupTest, unittest.TestCase): + + # @unittest.skip("skip") + def test_basic_default_search_path(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pgpro_edition() " + "RETURNS text " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + # @unittest.skip("skip") + def test_basic_backup_modified_search_path(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_control_checkpoint(OUT timeline_id integer, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_proc(OUT proname name, OUT dummy integer) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE '% vulnerable!', 'pg_probackup'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()") + + self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertFalse( + 'pg_probackup vulnerable!' in log_content) + + # @unittest.skip("skip") + def test_basic_checkdb_modified_search_path(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + node.slow_start() + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_database(OUT datname name, OUT oid oid, OUT dattablespace oid) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_database AS SELECT * FROM public.pg_database()") + + node.safe_psql( + 'postgres', + "CREATE FUNCTION public.pg_extension(OUT extname name, OUT extnamespace oid, OUT extversion text) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE FUNCTION public.pg_namespace(OUT oid oid, OUT nspname name) " + "RETURNS record " + "AS $$ " + "BEGIN " + " RAISE 'pg_probackup vulnerable!'; " + "END " + "$$ LANGUAGE plpgsql; " + "CREATE VIEW public.pg_extension AS SELECT * FROM public.pg_extension();" + "CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();" + ) + + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + self.assertEqual( + 1, 0, + "Expecting Error because amcheck{,_next} not installed\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) diff --git a/tests/Readme.md b/tests/Readme.md index c1dd9a63d..11c5272f9 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -1,7 +1,11 @@ -[см wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) +****[see wiki](https://confluence.postgrespro.ru/display/DEV/pg_probackup) ``` -Note: For now these are works on Linux and "kinda" works on Windows +Note: For now these tests work on Linux and "kinda" work on Windows +``` + +``` +Note: tests require python3 to work properly. ``` ``` @@ -26,15 +30,40 @@ Specify path to pg_probackup binary file. By default tests use /proc/sys/kernel/yama/ptrace_scope + pip install testgres export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` + +# Troubleshooting FAQ + +## Python tests failure +### 1. Could not open extension "..." +``` +testgres.exceptions.QueryException ERROR: could not open extension control file "/share/extension/amcheck.control": No such file or directory +``` + +#### Solution: + +You have no `/contrib/...` extension installed, please do + +```commandline +cd +make install-world +``` diff --git a/tests/__init__.py b/tests/__init__.py index dbf84feea..c8d2c70c3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,12 +1,13 @@ import unittest import os -from . import init, merge, option, show, compatibility, \ - backup, delete, delta, restore, validate, \ - retention, pgpro560, pgpro589, pgpro2068, false_positive, replica, \ - compression, page, ptrack, archive, exclude, cfs_backup, cfs_restore, \ - cfs_validate_backup, auth_test, time_stamp, snapfs, logging, \ - locking, remote, external, config, checkdb, set_backup, incr_restore +from . import init_test, merge_test, option_test, show_test, compatibility_test, \ + backup_test, delete_test, delta_test, restore_test, validate_test, \ + retention_test, pgpro560_test, pgpro589_test, pgpro2068_test, false_positive_test, replica_test, \ + compression_test, page_test, ptrack_test, archive_test, exclude_test, cfs_backup_test, cfs_restore_test, \ + cfs_validate_backup_test, auth_test, time_stamp_test, logging_test, \ + locking_test, remote_test, external_test, config_test, checkdb_test, set_backup_test, incr_restore_test, \ + catchup_test, CVE_2018_1058_test, time_consuming_test def load_tests(loader, tests, pattern): @@ -18,42 +19,50 @@ def load_tests(loader, tests, pattern): if 'PG_PROBACKUP_PTRACK' in os.environ: if os.environ['PG_PROBACKUP_PTRACK'] == 'ON': - suite.addTests(loader.loadTestsFromModule(ptrack)) + suite.addTests(loader.loadTestsFromModule(ptrack_test)) -# suite.addTests(loader.loadTestsFromModule(auth_test)) - suite.addTests(loader.loadTestsFromModule(archive)) - suite.addTests(loader.loadTestsFromModule(backup)) - suite.addTests(loader.loadTestsFromModule(compatibility)) - suite.addTests(loader.loadTestsFromModule(checkdb)) - suite.addTests(loader.loadTestsFromModule(config)) -# suite.addTests(loader.loadTestsFromModule(cfs_backup)) -# suite.addTests(loader.loadTestsFromModule(cfs_restore)) -# suite.addTests(loader.loadTestsFromModule(cfs_validate_backup)) - suite.addTests(loader.loadTestsFromModule(compression)) - suite.addTests(loader.loadTestsFromModule(delete)) - suite.addTests(loader.loadTestsFromModule(delta)) - suite.addTests(loader.loadTestsFromModule(exclude)) - suite.addTests(loader.loadTestsFromModule(external)) - suite.addTests(loader.loadTestsFromModule(false_positive)) - suite.addTests(loader.loadTestsFromModule(init)) - suite.addTests(loader.loadTestsFromModule(incr_restore)) - suite.addTests(loader.loadTestsFromModule(locking)) - suite.addTests(loader.loadTestsFromModule(logging)) - suite.addTests(loader.loadTestsFromModule(merge)) - suite.addTests(loader.loadTestsFromModule(option)) - suite.addTests(loader.loadTestsFromModule(page)) - suite.addTests(loader.loadTestsFromModule(pgpro560)) - suite.addTests(loader.loadTestsFromModule(pgpro589)) - suite.addTests(loader.loadTestsFromModule(pgpro2068)) - suite.addTests(loader.loadTestsFromModule(remote)) - suite.addTests(loader.loadTestsFromModule(replica)) - suite.addTests(loader.loadTestsFromModule(restore)) - suite.addTests(loader.loadTestsFromModule(retention)) - suite.addTests(loader.loadTestsFromModule(set_backup)) - suite.addTests(loader.loadTestsFromModule(show)) - suite.addTests(loader.loadTestsFromModule(snapfs)) - suite.addTests(loader.loadTestsFromModule(time_stamp)) - suite.addTests(loader.loadTestsFromModule(validate)) + # PG_PROBACKUP_LONG section for tests that are long + # by design e.g. they contain loops, sleeps and so on + if 'PG_PROBACKUP_LONG' in os.environ: + if os.environ['PG_PROBACKUP_LONG'] == 'ON': + suite.addTests(loader.loadTestsFromModule(time_consuming_test)) + + suite.addTests(loader.loadTestsFromModule(auth_test)) + suite.addTests(loader.loadTestsFromModule(archive_test)) + suite.addTests(loader.loadTestsFromModule(backup_test)) + suite.addTests(loader.loadTestsFromModule(catchup_test)) + if 'PGPROBACKUPBIN_OLD' in os.environ and os.environ['PGPROBACKUPBIN_OLD']: + suite.addTests(loader.loadTestsFromModule(compatibility_test)) + suite.addTests(loader.loadTestsFromModule(checkdb_test)) + suite.addTests(loader.loadTestsFromModule(config_test)) + suite.addTests(loader.loadTestsFromModule(cfs_backup_test)) + suite.addTests(loader.loadTestsFromModule(cfs_restore_test)) + suite.addTests(loader.loadTestsFromModule(cfs_validate_backup_test)) + suite.addTests(loader.loadTestsFromModule(compression_test)) + suite.addTests(loader.loadTestsFromModule(delete_test)) + suite.addTests(loader.loadTestsFromModule(delta_test)) + suite.addTests(loader.loadTestsFromModule(exclude_test)) + suite.addTests(loader.loadTestsFromModule(external_test)) + suite.addTests(loader.loadTestsFromModule(false_positive_test)) + suite.addTests(loader.loadTestsFromModule(init_test)) + suite.addTests(loader.loadTestsFromModule(incr_restore_test)) + suite.addTests(loader.loadTestsFromModule(locking_test)) + suite.addTests(loader.loadTestsFromModule(logging_test)) + suite.addTests(loader.loadTestsFromModule(merge_test)) + suite.addTests(loader.loadTestsFromModule(option_test)) + suite.addTests(loader.loadTestsFromModule(page_test)) + suite.addTests(loader.loadTestsFromModule(pgpro560_test)) + suite.addTests(loader.loadTestsFromModule(pgpro589_test)) + suite.addTests(loader.loadTestsFromModule(pgpro2068_test)) + suite.addTests(loader.loadTestsFromModule(remote_test)) + suite.addTests(loader.loadTestsFromModule(replica_test)) + suite.addTests(loader.loadTestsFromModule(restore_test)) + suite.addTests(loader.loadTestsFromModule(retention_test)) + suite.addTests(loader.loadTestsFromModule(set_backup_test)) + suite.addTests(loader.loadTestsFromModule(show_test)) + suite.addTests(loader.loadTestsFromModule(time_stamp_test)) + suite.addTests(loader.loadTestsFromModule(validate_test)) + suite.addTests(loader.loadTestsFromModule(CVE_2018_1058_test)) return suite diff --git a/tests/archive.py b/tests/archive_test.py similarity index 75% rename from tests/archive.py rename to tests/archive_test.py index 01ff5c062..00fd1f592 100644 --- a/tests/archive.py +++ b/tests/archive_test.py @@ -3,6 +3,7 @@ import gzip import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException +from .helpers.data_helpers import tail_file from datetime import datetime, timedelta import subprocess from sys import exit @@ -10,19 +11,15 @@ from distutils.dir_util import copy_tree -module_name = 'archive' - - class ArchiveTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_pgpro434_1(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -39,7 +36,7 @@ def test_pgpro434_1(self): "md5(repeat(i::text,10))::tsvector as tsvector from " "generate_series(0,100) i") - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node) node.cleanup() @@ -62,10 +59,8 @@ def test_pgpro434_1(self): node.slow_start() self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + result, node.table_checksum("t_heap"), 'data after restore not equal to original data') - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure @@ -74,15 +69,19 @@ def test_pgpro434_2(self): Check that timelines are correct. WAITING PGPRO-1053 for --immediate """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'} ) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -154,7 +153,7 @@ def test_pgpro434_2(self): backup_id = self.backup_node(backup_dir, 'node', node) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node.safe_psql( "postgres", "insert into t_heap select 100503 as id, md5(i::text) as text, " @@ -206,26 +205,20 @@ def test_pgpro434_2(self): "select exists(select 1 from t_heap where id > 100500)")[0][0], 'data after restore not equal to original data') - self.assertEqual( - result, - node.safe_psql( - "postgres", - "SELECT * FROM t_heap"), + self.assertEqual(result, node.table_checksum("t_heap"), 'data after restore not equal to original data') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro434_3(self): """ Check pg_stop_backup_timeout, needed backup_timeout Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -242,6 +235,7 @@ def test_pgpro434_3(self): "--log-level-file=LOG"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -275,19 +269,17 @@ def test_pgpro434_3(self): log_content, 'PostgreSQL crashed because of a failed assert') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro434_4(self): """ Check pg_stop_backup_timeout, libpq-timeout requested. Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -304,10 +296,11 @@ def test_pgpro434_4(self): "--log-level-file=info"], gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - self.set_auto_conf(node, {'archive_command': "'exit 1'"}) + self.set_auto_conf(node, {'archive_command': 'exit 1'}) node.reload() os.environ["PGAPPNAME"] = "foo" @@ -316,12 +309,15 @@ def test_pgpro434_4(self): "postgres", "SELECT pid " "FROM pg_stat_activity " - "WHERE application_name = 'pg_probackup'").rstrip() + "WHERE application_name = 'pg_probackup'").decode('utf-8').rstrip() os.environ["PGAPPNAME"] = "pg_probackup" postgres_gdb = self.gdb_attach(pid) - postgres_gdb.set_breakpoint('do_pg_stop_backup') + if self.get_version(node) < 150000: + postgres_gdb.set_breakpoint('do_pg_stop_backup') + else: + postgres_gdb.set_breakpoint('do_pg_backup_stop') postgres_gdb.continue_execution_until_running() gdb.continue_execution_until_exit() @@ -331,9 +327,14 @@ def test_pgpro434_4(self): with open(log_file, 'r') as f: log_content = f.read() - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + log_content) + else: + self.assertIn( + "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: @@ -344,16 +345,12 @@ def test_pgpro434_4(self): log_content, 'PostgreSQL crashed because of a failed assert') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_file_exists(self): """Archive-push if file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -371,7 +368,7 @@ def test_archive_push_file_exists(self): filename = '000000010000000000000001' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") f.flush() f.close() @@ -387,26 +384,31 @@ def test_archive_push_file_exists(self): self.switch_wal_segment(node) sleep(1) - with open(log_file, 'r') as f: - log_content = f.read() + log = tail_file(log_file, linetimeout=30, totaltimeout=120, + collect=True) + log.wait(contains = 'The failed archive command was') + self.assertIn( 'LOG: archive command failed with exit code 1', - log_content) + log.content) self.assertIn( 'DETAIL: The failed archive command was:', - log_content) + log.content) self.assertIn( 'pg_probackup archive-push WAL file', - log_content) + log.content) self.assertIn( 'WAL file already exists in archive with different checksum', - log_content) + log.content) self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) + 'pg_probackup archive-push completed successfully', log.content) + + # btw check that console coloring codes are not slipped into log file + self.assertNotIn('[0m', log.content) if self.get_version(node) < 100000: wal_src = os.path.join( @@ -423,25 +425,16 @@ def test_archive_push_file_exists(self): shutil.copyfile(wal_src, file) self.switch_wal_segment(node) - sleep(5) - with open(log_file, 'r') as f: - log_content = f.read() - - self.assertIn( - 'pg_probackup archive-push completed successfully', - log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) + log.stop_collect() + log.wait(contains = 'pg_probackup archive-push completed successfully') # @unittest.skip("skip") def test_archive_push_file_exists_overwrite(self): """Archive-push if file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'checkpoint_timeout': '30s'}) @@ -458,7 +451,7 @@ def test_archive_push_file_exists_overwrite(self): filename = '000000010000000000000001' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blablablaadssaaaaaaaaaaaaaaa") f.flush() f.close() @@ -474,50 +467,42 @@ def test_archive_push_file_exists_overwrite(self): self.switch_wal_segment(node) sleep(1) - with open(log_file, 'r') as f: - log_content = f.read() + log = tail_file(log_file, linetimeout=30, collect=True) + log.wait(contains = 'The failed archive command was') self.assertIn( - 'LOG: archive command failed with exit code 1', log_content) + 'LOG: archive command failed with exit code 1', log.content) self.assertIn( - 'DETAIL: The failed archive command was:', log_content) + 'DETAIL: The failed archive command was:', log.content) self.assertIn( - 'pg_probackup archive-push WAL file', log_content) + 'pg_probackup archive-push WAL file', log.content) self.assertNotIn( 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) + 'different checksum, overwriting', log.content) self.assertIn( 'WAL file already exists in archive with ' - 'different checksum', log_content) + 'different checksum', log.content) self.assertNotIn( - 'pg_probackup archive-push completed successfully', log_content) + 'pg_probackup archive-push completed successfully', log.content) self.set_archiving(backup_dir, 'node', node, overwrite=True) node.reload() self.switch_wal_segment(node) - sleep(5) - with open(log_file, 'r') as f: - log_content = f.read() - self.assertTrue( - 'pg_probackup archive-push completed successfully' in log_content, - 'Expecting messages about successfull execution archive_command') + log.drop_content() + log.wait(contains = 'pg_probackup archive-push completed successfully') self.assertIn( 'WAL file already exists in archive with ' - 'different checksum, overwriting', log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) + 'different checksum, overwriting', log.content) # @unittest.skip("skip") def test_archive_push_partial_file_exists(self): """Archive-push if stale '.part' file exists""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -538,7 +523,7 @@ def test_archive_push_partial_file_exists(self): xid = node.safe_psql( "postgres", - "INSERT INTO t1 VALUES (1) RETURNING (xmin)").rstrip() + "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() if self.get_version(node) < 100000: filename_orig = node.safe_psql( @@ -551,6 +536,8 @@ def test_archive_push_partial_file_exists(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = filename_orig.decode('utf-8') + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: @@ -561,7 +548,7 @@ def test_archive_push_partial_file_exists(self): file = os.path.join(wals_dir, filename) # emulate stale .part file - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblah") f.flush() f.close() @@ -589,16 +576,12 @@ def test_archive_push_partial_file_exists(self): 'Reusing stale temp WAL file', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_push_part_file_exists_not_stale(self): """Archive-push if .part file exists and it is not stale""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -628,6 +611,8 @@ def test_archive_push_part_file_exists_not_stale(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = filename_orig.decode('utf-8') + # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: @@ -637,7 +622,7 @@ def test_archive_push_part_file_exists_not_stale(self): filename = filename_orig + '.part' file = os.path.join(wals_dir, filename) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblah") f.flush() f.close() @@ -645,7 +630,7 @@ def test_archive_push_part_file_exists_not_stale(self): self.switch_wal_segment(node) sleep(30) - with open(file, 'a') as f: + with open(file, 'a+b') as f: f.write(b"blahblahblahblah") f.flush() f.close() @@ -667,9 +652,6 @@ def test_archive_push_part_file_exists_not_stale(self): # 'is not stale', # log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_replica_archive(self): @@ -678,10 +660,9 @@ def test_replica_archive(self): turn it into replica, set replica with archiving, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -689,13 +670,17 @@ def test_replica_archive(self): 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) # ADD INSTANCE 'MASTER' self.add_instance(backup_dir, 'master', master) master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.psql( @@ -705,7 +690,7 @@ def test_replica_archive(self): "from generate_series(0,2560) i") self.backup_node(backup_dir, 'master', master, options=['--stream']) - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # Settings for Replica self.restore_node(backup_dir, 'master', replica) @@ -716,7 +701,7 @@ def test_replica_archive(self): replica.slow_start(replica=True) # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -724,10 +709,10 @@ def test_replica_archive(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'replica', replica, @@ -744,14 +729,14 @@ def test_replica_archive(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) self.set_auto_conf(node, {'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, make PAGE backup from replica, @@ -759,11 +744,11 @@ def test_replica_archive(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,80680) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.wait_until_replica_catch_with_master(master, replica) @@ -790,12 +775,9 @@ def test_replica_archive(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_master_and_replica_parallel_archiving(self): @@ -805,17 +787,21 @@ def test_master_and_replica_parallel_archiving(self): set replica with archiving, make archive backup from replica, make archive backup from master """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'} ) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -833,7 +819,7 @@ def test_master_and_replica_parallel_archiving(self): # TAKE FULL ARCHIVE BACKUP FROM MASTER self.backup_node(backup_dir, 'master', master) # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # GET PHYSICAL CONTENT FROM MASTER pgdata_master = self.pgdata_content(master.data_dir) @@ -851,7 +837,7 @@ def test_master_and_replica_parallel_archiving(self): replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) master.psql( @@ -879,9 +865,6 @@ def test_master_and_replica_parallel_archiving(self): self.assertEqual( 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_basic_master_and_replica_concurrent_archiving(self): @@ -891,18 +874,24 @@ def test_basic_master_and_replica_concurrent_archiving(self): set replica with archiving, make sure that archiving on both node is working. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if self.pg_config_version < self.version_to_num('9.6.0'): + self.skipTest('You need PostgreSQL >= 9.6 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'archive_timeout': '10s'}) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.init_pb(backup_dir) @@ -922,7 +911,7 @@ def test_basic_master_and_replica_concurrent_archiving(self): # TAKE FULL ARCHIVE BACKUP FROM MASTER self.backup_node(backup_dir, 'master', master) # GET LOGICAL CONTENT FROM MASTER - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # GET PHYSICAL CONTENT FROM MASTER pgdata_master = self.pgdata_content(master.data_dir) @@ -941,12 +930,12 @@ def test_basic_master_and_replica_concurrent_archiving(self): replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") @@ -975,17 +964,91 @@ def test_basic_master_and_replica_concurrent_archiving(self): self.backup_node(backup_dir, 'master', master) self.backup_node(backup_dir, 'master', replica) - # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[master, replica]) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_concurrent_archiving(self): + """ + Concurrent archiving from master, replica and cascade replica + https://github.com/postgrespro/pg_probackup/issues/327 + + For PG >= 11 it is expected to pass this test + """ + + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', master) + self.set_archiving(backup_dir, 'node', master, replica=True) + master.slow_start() + + master.pgbench_init(scale=10) + + # TAKE FULL ARCHIVE BACKUP FROM MASTER + self.backup_node(backup_dir, 'node', master) + + # Settings for Replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + self.set_replica(master, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + replica.slow_start(replica=True) + + # create cascade replicas + replica1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica1')) + replica1.cleanup() + + # Settings for casaced replica + self.restore_node(backup_dir, 'node', replica1) + self.set_replica(replica, replica1, synchronous=False) + self.set_auto_conf(replica1, {'port': replica1.port}) + replica1.slow_start(replica=True) + + # Take full backup from master + self.backup_node(backup_dir, 'node', master) + + pgbench = master.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '30', '-c', '1']) + + # Take several incremental backups from master + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + + pgbench.wait() + pgbench.stdout.close() + + with open(os.path.join(master.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) + + with open(os.path.join(replica1.logs_dir, 'postgresql.log'), 'r') as f: + log_content = f.read() + self.assertNotIn('different checksum', log_content) # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog(self): """Test backup with pg_receivexlog wal delivary method""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1032,7 +1095,7 @@ def test_archive_pg_receivexlog(self): node, backup_type='page' ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.validate_pb(backup_dir) # Check data correctness @@ -1042,23 +1105,19 @@ def test_archive_pg_receivexlog(self): self.assertEqual( result, - node.safe_psql( - "postgres", "SELECT * FROM t_heap" - ), + node.table_checksum("t_heap"), 'data after restore not equal to original data') # Clean after yourself pg_receivexlog.kill() - self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog_compression_pg10(self): """Test backup with pg_receivewal compressed wal delivary method""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1068,7 +1127,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): self.add_instance(backup_dir, 'node', node) node.slow_start() if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') + self.skipTest('You need PostgreSQL >= 10 for this test') else: pg_receivexlog_path = self.get_bin_path('pg_receivewal') @@ -1103,7 +1162,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): backup_dir, 'node', node, backup_type='page' ) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") self.validate_pb(backup_dir) # Check data correctness @@ -1112,12 +1171,11 @@ def test_archive_pg_receivexlog_compression_pg10(self): node.slow_start() self.assertEqual( - result, node.safe_psql("postgres", "SELECT * FROM t_heap"), + result, node.table_checksum("t_heap"), 'data after restore not equal to original data') # Clean after yourself pg_receivexlog.kill() - self.del_test_dir(module_name, fname) # @unittest.expectedFailure # @unittest.skip("skip") @@ -1138,20 +1196,17 @@ def test_archive_catalog(self): ARCHIVE master: t1 -Z1--Z2--- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1180,7 +1235,7 @@ def test_archive_catalog(self): backup_dir, 'master', master, backup_type='page') replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) @@ -1449,8 +1504,6 @@ def test_archive_catalog(self): self.assertEqual(timeline_2['parent-tli'], 1) self.assertEqual(timeline_1['parent-tli'], 0) - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_catalog_1(self): @@ -1458,19 +1511,17 @@ def test_archive_catalog_1(self): double segment - compressed and not """ if not self.archive_compress: - return self.fail( - 'You need to enable ARCHIVE_COMPRESSION for this test to run') + self.skipTest('You need to enable ARCHIVE_COMPRESSION ' + 'for this test to run') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1505,8 +1556,6 @@ def test_archive_catalog_1(self): '000000010000000000000001') self.assertEqual(timeline['status'], 'OK') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_catalog_2(self): @@ -1514,19 +1563,17 @@ def test_archive_catalog_2(self): double segment - compressed and not """ if not self.archive_compress: - return self.fail( - 'You need to enable ARCHIVE_COMPRESSION for this test to run') + self.skipTest('You need to enable ARCHIVE_COMPRESSION ' + 'for this test to run') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1563,8 +1610,6 @@ def test_archive_catalog_2(self): '000000010000000000000002') self.assertEqual(timeline['status'], 'OK') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_options(self): @@ -1572,10 +1617,12 @@ def test_archive_options(self): check that '--archive-host', '--archive-user', '--archiver-port' and '--restore-command' are working as expected. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1602,7 +1649,7 @@ def test_archive_options(self): ]) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1626,7 +1673,7 @@ def test_archive_options(self): recovery_content = f.read() self.assertIn( - "restore_command = '{0} archive-get -B {1} --instance {2} " + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " "--remote-port=22 --remote-user={3}'".format( self.probackup_path, backup_dir, 'node', self.user), @@ -1638,8 +1685,6 @@ def test_archive_options(self): 'postgres', 'select 1') - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_options_1(self): @@ -1647,10 +1692,9 @@ def test_archive_options_1(self): check that '--archive-host', '--archive-user', '--archiver-port' and '--restore-command' are working as expected with set-config """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1677,7 +1721,7 @@ def test_archive_options_1(self): self.restore_node(backup_dir, 'node', node) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1693,7 +1737,7 @@ def test_archive_options_1(self): self.restore_node( backup_dir, 'node', node, options=[ - '--restore-command=none'.format(wal_dir), + '--restore-command=none', '--archive-host=localhost1', '--archive-port=23', '--archive-user={0}'.format(self.user) @@ -1703,13 +1747,129 @@ def test_archive_options_1(self): recovery_content = f.read() self.assertIn( - "restore_command = '{0} archive-get -B {1} --instance {2} " + "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " "--remote-port=23 --remote-user={3}'".format( self.probackup_path, backup_dir, 'node', self.user), recovery_content) - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_undefined_wal_file_path(self): + """ + check that archive-push works correct with undefined + --wal-file-path + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + if os.name == 'posix': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node') + elif os.name == 'nt': + archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( + self.probackup_path, backup_dir, 'node').replace("\\","\\\\") + else: + self.assertTrue(False, 'Unexpected os family') + + self.set_auto_conf( + node, + {'archive_command': archive_command}) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_intermediate_archiving(self): + """ + check that archive-push works correct with --wal-file-path setting by user + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + node_pg_options = {} + if node.major_version >= 13: + node_pg_options['wal_keep_size'] = '0MB' + else: + node_pg_options['wal_keep_segments'] = '0' + self.set_auto_conf(node, node_pg_options) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'intermediate_dir') + shutil.rmtree(wal_dir, ignore_errors=True) + os.makedirs(wal_dir) + if os.name == 'posix': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) + elif os.name == 'nt': + self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) + else: + self.assertTrue(False, 'Unexpected os family') + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + wal_segment = '000000010000000000000001' + + self.run_pb(["archive-push", "-B", backup_dir, + "--instance=node", "-D", node.data_dir, + "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment]) + + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) + + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_waldir_outside_pgdata_archiving(self): + """ + check that archive-push works correct with symlinked waldir + """ + if self.pg_config_version < self.version_to_num('10.0'): + self.skipTest( + 'Skipped because waldir outside pgdata is supported since PG 10') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') + shutil.rmtree(external_wal_dir, ignore_errors=True) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0, 10) i") + self.switch_wal_segment(node) + + # check + self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') # @unittest.skip("skip") # @unittest.expectedFailure @@ -1717,13 +1877,11 @@ def test_hexadecimal_timeline(self): """ Check that timelines are correct. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1769,9 +1927,6 @@ def test_hexadecimal_timeline(self): '0000000D000000000000001C', tli13['max-segno']) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_archiving_and_slots(self): @@ -1779,14 +1934,12 @@ def test_archiving_and_slots(self): Check that archiving don`t break slot guarantee. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'checkpoint_timeout': '30s', 'max_wal_size': '64MB'}) @@ -1834,15 +1987,11 @@ def test_archiving_and_slots(self): exit(1) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_push_sanity(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1864,7 +2013,7 @@ def test_archive_push_sanity(self): self.backup_node(backup_dir, 'node', node) with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() + postgres_log_content = cleanup_ptrack(f.read()) # print(postgres_log_content) # make sure that .backup file is not compressed @@ -1872,14 +2021,14 @@ def test_archive_push_sanity(self): self.assertNotIn('WARNING', postgres_log_content) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( backup_dir, 'node', replica, data_dir=replica.data_dir, options=['-R']) - #self.set_archiving(backup_dir, 'replica', replica, replica=True) + # self.set_archiving(backup_dir, 'replica', replica, replica=True) self.set_auto_conf(replica, {'port': replica.port}) self.set_auto_conf(replica, {'archive_mode': 'always'}) self.set_auto_conf(replica, {'hot_standby': 'on'}) @@ -1892,70 +2041,65 @@ def test_archive_push_sanity(self): replica.promote() replica.pgbench_init(scale=10) - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - replica_log_content = f.read() + log = tail_file(os.path.join(replica.logs_dir, 'postgresql.log'), + collect=True) + log.wait(regex=r"pushing file.*history") + log.wait(contains='archive-push completed successfully') + log.wait(regex=r"pushing file.*partial") + log.wait(contains='archive-push completed successfully') # make sure that .partial file is not compressed - self.assertNotIn('.partial.gz', replica_log_content) + self.assertNotIn('.partial.gz', log.content) # make sure that .history file is not compressed - self.assertNotIn('.history.gz', replica_log_content) - self.assertNotIn('WARNING', replica_log_content) + self.assertNotIn('.history.gz', log.content) + + replica.stop() + log.wait_shutdown() + + self.assertNotIn('WARNING', cleanup_ptrack(log.content)) output = self.show_archive( backup_dir, 'node', as_json=False, as_text=True, - options=['--log-level-console=VERBOSE']) + options=['--log-level-console=INFO']) self.assertNotIn('WARNING', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_archive_pg_receivexlog_partial_handling(self): """check that archive-get delivers .partial and .gz.partial files""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '10s'}) + initdb_params=['--data-checksums']) + + if self.get_version(node) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - - self.restore_node( - backup_dir, 'node', replica, replica.data_dir, options=['-R']) - self.set_auto_conf(replica, {'port': replica.port}) - self.set_replica(node, replica) - - self.add_instance(backup_dir, 'replica', replica) - # self.set_archiving(backup_dir, 'replica', replica, replica=True) - - replica.slow_start(replica=True) - - if self.get_version(replica) < 100000: + if self.get_version(node) < 100000: + app_name = 'pg_receivexlog' pg_receivexlog_path = self.get_bin_path('pg_receivexlog') else: + app_name = 'pg_receivewal' pg_receivexlog_path = self.get_bin_path('pg_receivewal') cmdline = [ - pg_receivexlog_path, '-p', str(replica.port), '--synchronous', - '-D', os.path.join(backup_dir, 'wal', 'replica')] + pg_receivexlog_path, '-p', str(node.port), '--synchronous', + '-D', os.path.join(backup_dir, 'wal', 'node')] if self.archive_compress and node.major_version >= 10: cmdline += ['-Z', '1'] - pg_receivexlog = self.run_binary(cmdline, asynchronous=True) + env = self.test_env + env["PGAPPNAME"] = app_name + pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env=env) if pg_receivexlog.returncode: self.assertFalse( @@ -1963,8 +2107,12 @@ def test_archive_pg_receivexlog_partial_handling(self): 'Failed to start pg_receivexlog: {0}'.format( pg_receivexlog.communicate()[1])) + self.set_auto_conf(node, {'synchronous_standby_names': app_name}) + self.set_auto_conf(node, {'synchronous_commit': 'on'}) + node.reload() + # FULL - self.backup_node(backup_dir, 'replica', replica, options=['--stream']) + self.backup_node(backup_dir, 'node', node, options=['--stream']) node.safe_psql( "postgres", @@ -1974,7 +2122,7 @@ def test_archive_pg_receivexlog_partial_handling(self): # PAGE self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', options=['--stream']) + backup_dir, 'node', node, backup_type='page', options=['--stream']) node.safe_psql( "postgres", @@ -1982,48 +2130,33 @@ def test_archive_pg_receivexlog_partial_handling(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(1000000,2000000) i") + pg_receivexlog.kill() + node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( - backup_dir, 'replica', node_restored, - node_restored.data_dir, options=['--recovery-target=latest', '--recovery-target-action=promote']) + backup_dir, 'node', node_restored, node_restored.data_dir, + options=['--recovery-target=latest', '--recovery-target-action=promote']) self.set_auto_conf(node_restored, {'port': node_restored.port}) self.set_auto_conf(node_restored, {'hot_standby': 'off'}) - # it will set node_restored as warm standby. -# with open(os.path.join(node_restored.data_dir, "standby.signal"), 'w') as f: -# f.flush() -# f.close() - node_restored.slow_start() - result = node.safe_psql( - "postgres", - "select sum(id) from t_heap") - - result_new = node_restored.safe_psql( - "postgres", - "select sum(id) from t_heap") + result = node.table_checksum("t_heap") + result_new = node_restored.table_checksum("t_heap") self.assertEqual(result, result_new) - # Clean after yourself - pg_receivexlog.kill() - self.del_test_dir( - module_name, fname, [node, replica, node_restored]) - @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2071,9 +2204,7 @@ def test_multi_timeline_recovery_prefetching(self): node.slow_start() node.pgbench_init(scale=20) - result = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') + result = node.table_checksum("pgbench_accounts") node.stop() node.cleanup() @@ -2091,16 +2222,14 @@ def test_multi_timeline_recovery_prefetching(self): if node.major_version >= 12: node.append_conf( - 'probackup_recovery.conf', "restore_command = '{0}'".format(restore_command)) + 'postgresql.auto.conf', "restore_command = '{0}'".format(restore_command)) else: node.append_conf( 'recovery.conf', "restore_command = '{0}'".format(restore_command)) node.slow_start() - result_new = node.safe_psql( - 'postgres', - 'select * from pgbench_accounts') + result_new = node.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) @@ -2124,26 +2253,20 @@ def test_multi_timeline_recovery_prefetching(self): 'WAL segment 000000010000000000000006, prefetch state: 5/10', postgres_log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_get_batching_sanity(self): """ Make sure that batching works. .gz file is corrupted and uncompressed is not, check that both corruption detected and uncompressed file is used. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -2157,7 +2280,7 @@ def test_archive_get_batching_sanity(self): node.pgbench_init(scale=50) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( @@ -2176,7 +2299,7 @@ def test_archive_get_batching_sanity(self): restore_command += ' -j 2 --batch-size=10' - print(restore_command) + # print(restore_command) if node.major_version >= 12: self.set_auto_conf(replica, {'restore_command': restore_command}) @@ -2198,21 +2321,16 @@ def test_archive_get_batching_sanity(self): self.assertIn('prefetch state: 9/10', postgres_log_content) self.assertIn('prefetch state: 8/10', postgres_log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_archive_get_prefetch_corruption(self): """ Make sure that WAL corruption is detected. And --prefetch-dir is honored. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_keep_segments': '200'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2225,7 +2343,7 @@ def test_archive_get_prefetch_corruption(self): node.pgbench_init(scale=50) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node( @@ -2293,7 +2411,7 @@ def test_archive_get_prefetch_corruption(self): dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) shutil.copyfile(src_file, dst_file) - print(dst_file) + # print(dst_file) # corrupt file if files[-2].endswith('.gz'): @@ -2322,21 +2440,11 @@ def test_archive_get_prefetch_corruption(self): os.remove(os.path.join(replica.logs_dir, 'postgresql.log')) replica.slow_start(replica=True) - sleep(60) - - with open(os.path.join(replica.logs_dir, 'postgresql.log'), 'r') as f: - postgres_log_content = f.read() - - self.assertIn( - 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename), - postgres_log_content) - - self.assertIn( - 'LOG: restored log file "{0}" from archive'.format(filename), - postgres_log_content) - - # Clean after yourself - self.del_test_dir(module_name, fname) + prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) + restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log')) + tailer.wait(contains=prefetch_line) + tailer.wait(contains=restored_line) # @unittest.skip("skip") def test_archive_show_partial_files_handling(self): @@ -2344,10 +2452,9 @@ def test_archive_show_partial_files_handling(self): check that files with '.part', '.part.gz', '.partial' and '.partial.gz' siffixes are handled correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2377,6 +2484,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2399,6 +2508,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2421,6 +2532,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2443,6 +2556,8 @@ def test_archive_show_partial_files_handling(self): "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = filename.decode('utf-8') + self.switch_wal_segment(node) os.rename( @@ -2458,8 +2573,105 @@ def test_archive_show_partial_files_handling(self): 'WARNING', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_archive_empty_history_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/326 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + node.pgbench_init(scale=5) + + # FULL + self.backup_node(backup_dir, 'node', node) + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-action=promote']) + + # Node in timeline 2 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=2', + '--recovery-target-action=promote']) + + # Node in timeline 3 + node.slow_start() + + node.pgbench_init(scale=5) + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target=latest', + '--recovery-target-timeline=3', + '--recovery-target-action=promote']) + + # Node in timeline 4 + node.slow_start() + node.pgbench_init(scale=5) + + # Truncate history files + for tli in range(2, 5): + file = os.path.join( + backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) + with open(file, "w+") as f: + f.truncate() + + timelines = self.show_archive(backup_dir, 'node', options=['--log-level-file=INFO']) + + # check that all timelines has zero switchpoint + for timeline in timelines: + self.assertEqual(timeline['switchpoint'], '0/0') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + wal_dir = os.path.join(backup_dir, 'wal', 'node') + + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), + log_content) + self.assertIn( + 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), + log_content) + + +def cleanup_ptrack(log_content): + # PBCKP-423 - need to clean ptrack warning + ptrack_is_not = 'Ptrack 1.X is not supported anymore' + if ptrack_is_not in log_content: + lines = [line for line in log_content.splitlines() + if ptrack_is_not not in line] + log_content = "".join(lines) + return log_content + # TODO test with multiple not archived segments. # TODO corrupted file in archive. @@ -2484,4 +2696,4 @@ def test_archive_show_partial_files_handling(self): #t2 ---------------- # / #t1 -A-------- -# \ No newline at end of file +# diff --git a/tests/auth_test.py b/tests/auth_test.py index eca62316b..32cabc4a1 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -30,20 +30,23 @@ def test_backup_via_unprivileged_user(self): run a backups without EXECUTE rights on certain functions """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_senders': '2'} - ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + if self.ptrack: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") try: @@ -53,18 +56,39 @@ def test_backup_via_unprivileged_user(self): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_start_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_start_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: query failed: ERROR: permission denied " + "for function pg_backup_start", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_start_backup(text, boolean, boolean) TO backup;") + if self.get_version(node) < 150000: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_backup_start(text, boolean) TO backup;") + + if self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup") + else: + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") - time.sleep(1) try: self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -84,8 +108,6 @@ def test_backup_via_unprivileged_user(self): "GRANT EXECUTE ON FUNCTION" " pg_create_restore_point(text) TO backup;") - time.sleep(1) - try: self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -93,25 +115,32 @@ def test_backup_via_unprivileged_user(self): 1, 0, "Expecting Error due to missing grant on EXECUTE.") except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_stop_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: Query failed: ERROR: permission denied " + "for function pg_stop_backup", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: Query failed: ERROR: permission denied " + "for function pg_backup_stop", e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) if self.get_version(node) < self.version_to_num('10.0'): node.safe_psql( "postgres", "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - else: + elif self.get_version(node) < self.version_to_num('15.0'): node.safe_psql( "postgres", - "GRANT EXECUTE ON FUNCTION " - "pg_stop_backup(boolean, boolean) TO backup") - # Do this for ptrack backups + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup;") + else: node.safe_psql( "postgres", - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup") + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup;") self.backup_node( backup_dir, 'node', node, options=['-U', 'backup']) @@ -124,64 +153,29 @@ def test_backup_via_unprivileged_user(self): node.safe_psql( "test1", "create table t1 as select generate_series(0,100)") - if self.ptrack: - self.set_auto_conf(node, {'ptrack_enable': 'on'}) node.stop() node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on clearing ptrack_files.") - except ProbackupException as e: - self.assertIn( - "ERROR: must be superuser or replication role to clear ptrack files\n" - "query was: SELECT pg_catalog.pg_ptrack_clear()", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - time.sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on clearing ptrack_files.") - except ProbackupException as e: - self.assertIn( - "ERROR: must be superuser or replication role read ptrack files\n" - "query was: select pg_catalog.pg_ptrack_control_lsn()", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - node.safe_psql( "postgres", "ALTER ROLE backup REPLICATION") - time.sleep(1) - # FULL self.backup_node( - backup_dir, 'node', node, - options=['-U', 'backup']) + backup_dir, 'node', node, options=['-U', 'backup']) # PTRACK - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-U', 'backup']) - - # Clean after yourself - self.del_test_dir(module_name, fname) + if self.ptrack: + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-U', 'backup']) class AuthTest(unittest.TestCase): pb = None node = None + # TODO move to object scope, replace module_name @classmethod def setUpClass(cls): @@ -195,7 +189,10 @@ def setUpClass(cls): set_replication=True, initdb_params=['--data-checksums', '--auth-host=md5'] ) - modify_pg_hba(cls.node) + + cls.username = cls.pb.get_username() + + cls.modify_pg_hba(cls.node) cls.pb.init_pb(cls.backup_dir) cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) @@ -205,24 +202,54 @@ def setUpClass(cls): except StartNodeException: raise unittest.skip("Node hasn't started") - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_ptrack_clear() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_ptrack_get_and_clear(oid, oid) TO backup;") + if cls.pb.get_version(cls.node) < 100000: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + elif cls.pb.get_version(cls.node) < 150000: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + else: + cls.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + # TODO move to object scope, replace module_name @classmethod def tearDownClass(cls): cls.node.cleanup() @@ -230,12 +257,13 @@ def tearDownClass(cls): @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") def setUp(self): - self.cmd = ['backup', + self.pb_cmd = ['backup', '-B', self.backup_dir, '--instance', self.node.name, '-h', '127.0.0.1', '-p', str(self.node.port), '-U', 'backup', + '-d', 'postgres', '-b', 'FULL' ] @@ -255,44 +283,31 @@ def test_empty_password(self): """ Test case: PGPB_AUTH03 - zero password length """ try: self.assertIn("ERROR: no password supplied", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, '\0\r\n')) - ) + self.run_pb_with_auth('\0\r\n')) except (TIMEOUT, ExceptionPexpect) as e: self.fail(e.value) def test_wrong_password(self): """ Test case: PGPB_AUTH04 - incorrect password """ - try: - self.assertIn("password authentication failed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'wrong_password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.assertIn("password authentication failed", + self.run_pb_with_auth('wrong_password\r\n')) def test_right_password(self): """ Test case: PGPB_AUTH01 - correct password """ - try: - self.assertIn("completed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd, 'password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.assertIn("completed", + self.run_pb_with_auth('password\r\n')) def test_right_password_and_wrong_pgpass(self): """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - create_pgpass(self.pgpass_file, line) - try: - self.assertIn("completed", - str(run_pb_with_auth([self.pb.probackup_path] + self.cmd + ['-W'], 'password\r\n')) - ) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) + self.create_pgpass(self.pgpass_file, line) + self.assertIn("completed", + self.run_pb_with_auth('password\r\n', add_args=["-W"])) def test_ctrl_c_event(self): """ Test case: PGPB_AUTH02 - send interrupt signal """ try: - run_pb_with_auth([self.pb.probackup_path] + self.cmd, kill=True) + self.run_pb_with_auth(kill=True) except TIMEOUT: self.fail("Error: CTRL+C event ignored") @@ -300,91 +315,74 @@ def test_pgpassfile_env(self): """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - create_pgpass(path, line) + self.create_pgpass(path, line) self.pb.test_env["PGPASSFILE"] = path - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpass(self): """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - create_pgpass(self.pgpass_file, line) - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.create_pgpass(self.pgpass_file, line) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpassword(self): """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ self.pb.test_env["PGPASSWORD"] = "password" - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) def test_pgpassword_and_wrong_pgpass(self): """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - create_pgpass(self.pgpass_file, line) + self.create_pgpass(self.pgpass_file, line) self.pb.test_env["PGPASSWORD"] = "password" - try: - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - except ProbackupException as e: - self.fail(e) - + self.assertEqual( + "OK", + self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) -def run_pb_with_auth(cmd, password=None, kill=False): - try: - with spawn(" ".join(cmd), encoding='utf-8', timeout=10) as probackup: + def run_pb_with_auth(self, password=None, add_args = [], kill=False): + with spawn(self.pb.probackup_path, self.pb_cmd + add_args, encoding='utf-8', timeout=10) as probackup: result = probackup.expect(u"Password for user .*:", 5) if kill: probackup.kill(signal.SIGINT) elif result == 0: probackup.sendline(password) probackup.expect(EOF) - return probackup.before + return str(probackup.before) else: raise ExceptionPexpect("Other pexpect errors.") - except TIMEOUT: - raise TIMEOUT("Timeout error.") - except ExceptionPexpect: - raise ExceptionPexpect("Pexpect error.") - - -def modify_pg_hba(node): - """ - Description: - Add trust authentication for user postgres. Need for add new role and set grant. - :param node: - :return None: - """ - hba_conf = os.path.join(node.data_dir, "pg_hba.conf") - with open(hba_conf, 'r+') as fio: - data = fio.read() - fio.seek(0) - fio.write('host\tall\tpostgres\t127.0.0.1/0\ttrust\n' + data) - - -def create_pgpass(path, line): - with open(path, 'w') as passfile: - # host:port:db:username:password - passfile.write(line) - os.chmod(path, 0o600) + + + @classmethod + def modify_pg_hba(cls, node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data)) + + + def create_pgpass(self, path, line): + with open(path, 'w') as passfile: + # host:port:db:username:password + passfile.write(line) + os.chmod(path, 0o600) diff --git a/tests/backup.py b/tests/backup_test.py similarity index 59% rename from tests/backup.py rename to tests/backup_test.py index 73eb21022..dc60228b5 100644 --- a/tests/backup.py +++ b/tests/backup_test.py @@ -1,43 +1,88 @@ import unittest import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import re +from time import sleep, time +from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException import shutil from distutils.dir_util import copy_tree -from testgres import ProcessType +from testgres import ProcessType, QueryException +import subprocess -module_name = 'backup' +class BackupTest(ProbackupTest, unittest.TestCase): + def test_full_backup(self): + """ + Just test full backup with at least two segments + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) -class BackupTest(ProbackupTest, unittest.TestCase): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + def test_full_backup_stream(self): + """ + Just test full backup with at least two segments in stream mode + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, + options=["--stream"]) + + out = self.validate_pb(backup_dir, 'node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - backup_id = self.backup_node(backup_dir, 'node', node) + full_backup_id = self.backup_node(backup_dir, 'node', node) show_backup = self.show_pb(backup_dir, 'node')[0] self.assertEqual(show_backup['status'], "OK") @@ -46,7 +91,7 @@ def test_backup_modes_archive(self): # postmaster.pid and postmaster.opts shouldn't be copied excluded = True db_dir = os.path.join( - backup_dir, "backups", 'node', backup_id, "database") + backup_dir, "backups", 'node', full_backup_id, "database") for f in os.listdir(db_dir): if ( @@ -63,44 +108,39 @@ def test_backup_modes_archive(self): page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - # print self.show_pb(node) - show_backup = self.show_pb(backup_dir, 'node')[1] - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PAGE") + show_backup_1 = self.show_pb(backup_dir, 'node')[1] + self.assertEqual(show_backup_1['status'], "OK") + self.assertEqual(show_backup_1['backup-mode'], "PAGE") + + # delta backup mode + delta_backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="delta") + + show_backup_2 = self.show_pb(backup_dir, 'node')[2] + self.assertEqual(show_backup_2['status'], "OK") + self.assertEqual(show_backup_2['backup-mode'], "DELTA") # Check parent backup self.assertEqual( - backup_id, + full_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['id'])["parent-backup-id"]) - - # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - - show_backup = self.show_pb(backup_dir, 'node')[2] - self.assertEqual(show_backup['status'], "OK") - self.assertEqual(show_backup['backup-mode'], "PTRACK") + backup_id=show_backup_1['id'])["parent-backup-id"]) - # Check parent backup self.assertEqual( page_backup_id, self.show_pb( backup_dir, 'node', - backup_id=show_backup['id'])["parent-backup-id"]) - - # Clean after yourself - self.del_test_dir(module_name, fname) + backup_id=show_backup_2['id'])["parent-backup-id"]) # @unittest.skip("skip") def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -112,32 +152,19 @@ def test_smooth_checkpoint(self): self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") node.stop() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incremental_backup_without_full(self): - """page-level backup without validated full backup""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] + """page backup without validated full backup""" node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - try: self.backup_node(backup_dir, 'node', node, backup_type="page") # we should die here because exception is what we expect to happen @@ -148,24 +175,7 @@ def test_incremental_backup_without_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -174,22 +184,14 @@ def test_incremental_backup_without_full(self): self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") - self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], - "ERROR") - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -212,9 +214,7 @@ def test_incremental_backup_corrupt_full(self): except ProbackupException as e: self.assertTrue( "INFO: Validate backups of the instance 'node'" in e.message and - "WARNING: Backup file".format( - file) in e.message and - "is not found".format(file) in e.message and + "WARNING: Backup file" in e.message and "is not found" in e.message and "WARNING: Backup {0} data files are corrupted".format( backup_id) in e.message and "WARNING: Some backups are not valid" in e.message, @@ -231,7 +231,7 @@ def test_incremental_backup_corrupt_full(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -241,68 +241,19 @@ def test_incremental_backup_corrupt_full(self): self.assertEqual( self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_ptrack_threads(self): - """ptrack multi thread backup mode""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - ptrack_enable=True) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_ptrack_threads_stream(self): - """ptrack multi thread backup mode and stream""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] + def test_delta_threads_stream(self): + """delta multi thread backup mode and stream""" node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - ptrack_enable=True) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -310,24 +261,20 @@ def test_ptrack_threads_stream(self): self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") self.backup_node( backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) + backup_type="delta", options=["-j", "4", "--stream"]) self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -349,7 +296,7 @@ def test_page_detect_corruption(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() path = os.path.join(node.data_dir, heap_path) with open(path, "rb+", 0) as f: @@ -378,28 +325,23 @@ def test_page_detect_corruption(self): 'ERROR', "Backup Status should be ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_backup_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -416,7 +358,7 @@ def test_backup_detect_corruption(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -525,27 +467,23 @@ def test_backup_detect_corruption(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_detect_invalid_block_header(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -558,7 +496,7 @@ def test_backup_detect_invalid_block_header(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -666,27 +604,23 @@ def test_backup_detect_invalid_block_header(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_detect_missing_permissions(self): """make node, corrupt some page, check that backup failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if self.ptrack and node.major_version > 11: + if self.ptrack: node.safe_psql( "postgres", "create extension ptrack") @@ -699,7 +633,7 @@ def test_backup_detect_missing_permissions(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -807,22 +741,18 @@ def test_backup_detect_missing_permissions(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_backup_truncate_misaligned(self): """ make node, truncate file to size not even to BLCKSIZE, take backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -840,7 +770,7 @@ def test_backup_truncate_misaligned(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() heap_size = node.safe_psql( "postgres", @@ -858,19 +788,15 @@ def test_backup_truncate_misaligned(self): self.assertIn("WARNING: File", output) self.assertIn("invalid file size", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_tablespace_in_pgdata_pgpro_1376(self): """PGPRO-1376 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -922,7 +848,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): relfilenode = node.safe_psql( "postgres", "select 't_heap1'::regclass::oid" - ).rstrip() + ).decode('utf-8').rstrip() list = [] for root, dirs, files in os.walk(os.path.join( @@ -952,9 +878,6 @@ def test_tablespace_in_pgdata_pgpro_1376(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_tablespace_handling(self): """ @@ -963,19 +886,18 @@ def test_basic_tablespace_handling(self): check that restore with tablespace mapping will end with success """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) @@ -1011,7 +933,7 @@ def test_basic_tablespace_handling(self): tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1030,9 +952,10 @@ def test_basic_tablespace_handling(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -1064,22 +987,18 @@ def test_basic_tablespace_handling(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[node]) - # @unittest.skip("skip") def test_tablespace_handling_1(self): """ make node with tablespace A, take full backup, check that restore with tablespace mapping of tablespace B will end with error """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1099,7 +1018,7 @@ def test_tablespace_handling_1(self): options=["-j", "4", "--stream"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1122,22 +1041,18 @@ def test_tablespace_handling_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_tablespace_handling_2(self): """ make node without tablespaces, take full backup, check that restore with tablespace mapping will end with error """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1146,12 +1061,12 @@ def test_tablespace_handling_2(self): tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - self.backup_node( + backup_id = self.backup_node( backup_dir, 'node', node, backup_type="full", options=["-j", "4", "--stream"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -1168,22 +1083,20 @@ def test_tablespace_handling_2(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_full_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1205,11 +1118,11 @@ def test_drop_rel_during_full_backup(self): relative_path_1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() relative_path_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() absolute_path_1 = os.path.join(node.data_dir, relative_path_1) absolute_path_2 = os.path.join(node.data_dir, relative_path_2) @@ -1255,16 +1168,12 @@ def test_drop_rel_during_full_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_drop_db_during_full_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1323,16 +1232,14 @@ def test_drop_db_during_full_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_backup_delta(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1350,7 +1257,7 @@ def test_drop_rel_during_backup_delta(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() absolute_path = os.path.join(node.data_dir, relative_path) @@ -1392,19 +1299,16 @@ def test_drop_rel_during_backup_delta(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_rel_during_backup_page(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1418,7 +1322,7 @@ def test_drop_rel_during_backup_page(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() absolute_path = os.path.join(node.data_dir, relative_path) @@ -1443,6 +1347,7 @@ def test_drop_rel_during_backup_page(self): # File removed, we can proceed with backup gdb.continue_execution_until_exit() + gdb.kill() pgdata = self.pgdata_content(node.data_dir) @@ -1458,86 +1363,12 @@ def test_drop_rel_during_backup_page(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_drop_rel_during_backup_ptrack(self): - """""" - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,100) i") - - relative_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").rstrip() - - absolute_path = os.path.join(node.data_dir, relative_path) - - # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - - # PTRACK backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - gdb=True, options=['--log-level-file=LOG']) - - gdb.set_breakpoint('backup_files') - gdb.run_until_break() - - # REMOVE file - os.remove(absolute_path) - - # File removed, we can proceed with backup - gdb.continue_execution_until_exit() - - pgdata = self.pgdata_content(node.data_dir) - - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( - 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, - 'File "{0}" should be deleted but it`s not'.format(absolute_path)) - - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_persistent_slot_for_stream_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1562,20 +1393,18 @@ def test_persistent_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--slot=slot_1']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_temp_slot_for_stream_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'max_wal_size': '40MB'}) + pg_options={'max_wal_size': '40MB'}) + + if self.get_version(node) < self.version_to_num('10.0'): + self.skipTest('You need PostgreSQL >= 10 for this test') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1587,26 +1416,19 @@ def test_basic_temp_slot_for_stream_backup(self): backup_dir, 'node', node, options=['--stream', '--temp-slot']) - if self.get_version(node) < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - # FULL backup self.backup_node( backup_dir, 'node', node, options=['--stream', '--slot=slot_1', '--temp-slot']) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_backup_concurrent_drop_table(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1637,31 +1459,27 @@ def test_backup_concurrent_drop_table(self): gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() + gdb.kill() show_backup = self.show_pb(backup_dir, 'node')[0] self.assertEqual(show_backup['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname, nodes=[node]) - # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): """""" if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', '--wal-segsize=64'], pg_options={ - 'min_wal_size': '128MB', - 'autovacuum': 'off'}) + 'min_wal_size': '128MB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1731,16 +1549,14 @@ def test_pg_11_adjusted_wal_segment_size(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigint_handling(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1761,6 +1577,7 @@ def test_sigint_handling(self): gdb._execute('signal SIGINT') gdb.continue_execution_until_error() + gdb.kill() backup_id = self.show_pb(backup_dir, 'node')[0]['id'] @@ -1769,16 +1586,14 @@ def test_sigint_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigterm_handling(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1807,16 +1622,14 @@ def test_sigterm_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_sigquit_handling(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1844,16 +1657,12 @@ def test_sigquit_handling(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_table(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1879,19 +1688,15 @@ def test_drop_table(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_missing_file_permissions(self): """""" if os.name == 'nt': - return unittest.skip('Skipped because it is POSIX only test') + self.skipTest('Skipped because it is POSIX only test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1901,7 +1706,7 @@ def test_basic_missing_file_permissions(self): relative_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pg_class')").rstrip() + "select pg_relation_filepath('pg_class')").decode('utf-8').rstrip() full_path = os.path.join(node.data_dir, relative_path) @@ -1926,19 +1731,15 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 700) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_basic_missing_dir_permissions(self): """""" if os.name == 'nt': - return unittest.skip('Skipped because it is POSIX only test') + self.skipTest('Skipped because it is POSIX only test') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1967,18 +1768,14 @@ def test_basic_missing_dir_permissions(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - os.chmod(full_path, 700) - - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + os.rmdir(full_path) # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -1993,10 +1790,11 @@ def test_backup_with_least_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') - if self.ptrack and node.major_version >= 12: + if self.ptrack: node.safe_psql( "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") + "CREATE SCHEMA ptrack; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") # PG 9.5 if self.get_version(node) < 90600: @@ -2021,16 +1819,17 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -2054,10 +1853,12 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -2068,8 +1869,8 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -2092,8 +1893,10 @@ def test_backup_with_least_privileges_role(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -2104,43 +1907,60 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: - if node.major_version < 12: - for fname in [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)', - 'pg_catalog.pg_stop_backup()']: - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) - else: - fnames = [ - 'pg_catalog.ptrack_get_pagemapset(pg_lsn)', - 'pg_catalog.ptrack_init_lsn()' - ] + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} " - "TO backup".format(fname)) + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " + "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup self.backup_node( @@ -2175,9 +1995,6 @@ def test_backup_with_least_privileges_role(self): backup_dir, 'node', node, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing(self): """ @@ -2186,10 +2003,9 @@ def test_parent_choosing(self): PAGE1 <- CORRUPT FULL """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2239,9 +2055,6 @@ def test_parent_choosing(self): backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], full_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing_1(self): """ @@ -2250,10 +2063,9 @@ def test_parent_choosing_1(self): PAGE1 <- (missing) FULL """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2299,9 +2111,6 @@ def test_parent_choosing_1(self): backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], full_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing_2(self): """ @@ -2310,10 +2119,9 @@ def test_parent_choosing_2(self): PAGE1 <- OK FULL <- (missing) """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2349,7 +2157,7 @@ def test_parent_choosing_2(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - 'WARNING: Valid backup on current timeline 1 is not found' in e.message and + 'WARNING: Valid full backup on current timeline 1 is not found' in e.message and 'ERROR: Create new full backup before an incremental one' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -2359,19 +2167,15 @@ def test_parent_choosing_2(self): backup_dir, 'node')[2]['status'], 'ERROR') - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.skip("skip") + # @unittest.skip("skip") def test_backup_with_less_privileges_role(self): """ check permissions correctness from documentation: https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, initdb_params=['--data-checksums'], @@ -2391,7 +2195,7 @@ def test_backup_with_less_privileges_role(self): 'postgres', 'CREATE DATABASE backupdb') - if self.ptrack and node.major_version >= 12: + if self.ptrack: node.safe_psql( 'backupdb', 'CREATE EXTENSION ptrack') @@ -2400,7 +2204,6 @@ def test_backup_with_less_privileges_role(self): if self.get_version(node) < 90600: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2411,14 +2214,11 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2433,11 +2233,10 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " "COMMIT;" ) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', - "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2452,6 +2251,25 @@ def test_backup_with_less_privileges_role(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " "COMMIT;" ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "BEGIN; " + "CREATE ROLE backup WITH LOGIN; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " + "COMMIT;" + ) # enable STREAM backup node.safe_psql( @@ -2492,12 +2310,11 @@ def test_backup_with_less_privileges_role(self): datname='backupdb', options=['--stream', '-U', 'backup']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) return # Restore as replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -2562,22 +2379,16 @@ def test_backup_with_less_privileges_role(self): backup_dir, 'replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_issue_132(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2602,22 +2413,16 @@ def test_issue_132(self): exit(1) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_issue_132_1(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 @@ -2764,17 +2569,13 @@ def test_issue_132_1(self): 'INFO: Restore of backup {0} completed.'.format(delta_id), output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_note_sanity(self): """ test that adding note to backup works as expected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2802,18 +2603,14 @@ def test_note_sanity(self): 'note', backup_meta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_backup_made_by_newer_version(self): """incremental backup with parent made by newer version""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2855,21 +2652,102 @@ def test_parent_backup_made_by_newer_version(self): self.assertEqual( self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_issue_289(self): + """ + https://github.com/postgrespro/pg_probackup/issues/289 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + backup_type='page', options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "ERROR: Create new full backup before an incremental one", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + + # @unittest.skip("skip") + def test_issue_290(self): + """ + https://github.com/postgrespro/pg_probackup/issues/290 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + os.rmdir( + os.path.join(backup_dir, "wal", "node")) + + node.slow_start() + + try: + self.backup_node( + backup_dir, 'node', node, + options=['--archive-timeout=10s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because full backup is missing" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + "INFO: Wait for WAL segment", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "WAL archive directory is not accessible", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") @unittest.skip("skip") def test_issue_203(self): """ https://github.com/postgrespro/pg_probackup/issues/203 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2887,7 +2765,7 @@ def test_issue_203(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', @@ -2896,5 +2774,885 @@ def test_issue_203(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_issue_231(self): + """ + https://github.com/postgrespro/pg_probackup/issues/231 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + datadir = os.path.join(node.data_dir, '123') + + t0 = time() + while True: + with self.assertRaises(ProbackupException) as ctx: + self.backup_node(backup_dir, 'node', node) + pb1 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] + + t = time() + if int(pb1, 36) == int(t) and t % 1 < 0.5: + # ok, we have a chance to start next backup in same second + break + elif t - t0 > 20: + # Oops, we are waiting for too long. Looks like this runner + # is too slow. Lets skip the test. + self.skipTest("runner is too slow") + # sleep to the second's end so backup will not sleep for a second. + sleep(1 - t % 1) + + with self.assertRaises(ProbackupException) as ctx: + self.backup_node(backup_dir, 'node', node) + pb2 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] + + self.assertNotEqual(pb1, pb2) + + def test_incr_backup_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + # incremental restore into node1 + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + # @unittest.skip("skip") + def test_missing_wal_segment(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={'archive_timeout': '30s'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # get segments in pg_wal, sort then and remove all but the latest + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + + if node.major_version >= 10: + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + else: + pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog') + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, datname='backupdb', + options=['--stream', '--log-level-file=INFO'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('start_WAL_streaming') + gdb.run_until_break() + + # generate some more data + node.pgbench_init(scale=3) + + # remove redundant WAL segments in pg_wal + files = os.listdir(pg_wal_dir) + files.sort(reverse=True) + + # leave first two files in list + del files[:2] + for filename in files: + os.remove(os.path.join(pg_wal_dir, filename)) + + gdb.continue_execution_until_exit() + + self.assertIn( + 'unexpected termination of replication stream: ERROR: requested WAL segment', + gdb.output) + + self.assertIn( + 'has already been removed', + gdb.output) + + self.assertIn( + 'ERROR: Interrupted during waiting for WAL streaming', + gdb.output) + + self.assertIn( + 'WARNING: A backup is in progress, stopping it', + gdb.output) + + # TODO: check the same for PAGE backup + + # @unittest.skip("skip") + def test_missing_replication_permission(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) +# self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + sleep(2) + replica.promote() + + # Delta backup + try: + self.backup_node( + backup_dir, 'node', replica, backup_type='delta', + data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + # 9.5: ERROR: must be superuser or replication role to run a backup + # >=9.6: FATAL: must be superuser or replication role to start walsender + if self.pg_config_version < 160000: + self.assertRegex( + e.message, + "ERROR: must be superuser or replication role to run a backup|" + "FATAL: must be superuser or replication role to start walsender", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertRegex( + e.message, + "FATAL: permission denied to start WAL sender\n" + "DETAIL: Only roles with the REPLICATION", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_missing_replication_permission_1(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica) + replica.slow_start(replica=True) + + node.safe_psql( + 'postgres', + 'CREATE DATABASE backupdb') + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + # > 15 + else: + node.safe_psql( + 'backupdb', + "CREATE ROLE backup WITH LOGIN; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") + + replica.promote() + + # PAGE + output = self.backup_node( + backup_dir, 'node', replica, backup_type='page', + data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'], + return_id=False) + + self.assertIn( + 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', + output) + + # Messages before 14 + # 'WARNING: could not connect to database backupdb: FATAL: must be superuser or replication role to start walsender' + # Messages for >=14 + # 'WARNING: could not connect to database backupdb: connection to server on socket "/tmp/.s.PGSQL.30983" failed: FATAL: must be superuser or replication role to start walsender' + # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' + # OS-dependant messages: + # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (::1), port 12101 failed: Connection refused\n\tIs the server running on that host and accepting TCP/IP connections?\nconnection to server at "localhost" (127.0.0.1), port 12101 failed: FATAL: must be superuser or replication role to start walsender' + + if self.pg_config_version < 160000: + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb:[\s\S]*?' + r'FATAL: must be superuser or replication role to start walsender') + else: + self.assertRegex( + output, + r'WARNING: could not connect to database backupdb:[\s\S]*?' + r'FATAL: permission denied to start WAL sender') + + # @unittest.skip("skip") + def test_basic_backup_default_transaction_read_only(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'default_transaction_read_only': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + try: + node.safe_psql( + 'postgres', + 'create temp table t1()') + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because incremental backup should not be possible " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except QueryException as e: + self.assertIn( + "cannot execute CREATE TABLE in a read-only transaction", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream']) + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', options=['--stream']) + + # PAGE backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + # @unittest.skip("skip") + def test_backup_atexit(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + # Full backup in streaming mode + gdb = self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--log-level-file=VERBOSE'], gdb=True) + + # break at streaming start + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + sleep(1) + + self.assertEqual( + self.show_pb( + backup_dir, 'node')[0]['status'], 'ERROR') + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + #print(log_content) + self.assertIn( + 'WARNING: A backup is in progress, stopping it.', + log_content) + + if self.get_version(node) < 150000: + self.assertIn( + 'FROM pg_catalog.pg_stop_backup', + log_content) + else: + self.assertIn( + 'FROM pg_catalog.pg_backup_stop', + log_content) + + self.assertIn( + 'setting its status to ERROR', + log_content) + + # @unittest.skip("skip") + def test_pg_stop_backup_missing_permissions(self): + """""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=5) + + self.simple_bootstrap(node, 'backup') + + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') + elif self.get_version(node) < 150000: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') + else: + node.safe_psql( + 'postgres', + 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') + + + # Full backup in streaming mode + try: + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '-U', 'backup']) + # we should die here because exception is what we expect to happen + if self.get_version(node) < 150000: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_stop_backup " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertEqual( + 1, 0, + "Expecting Error because of missing permissions on pg_backup_stop " + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + if self.get_version(node) < 150000: + self.assertIn( + "ERROR: permission denied for function pg_stop_backup", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + else: + self.assertIn( + "ERROR: permission denied for function pg_backup_stop", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "query was: SELECT pg_catalog.txid_snapshot_xmax", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_start_time(self): + """Test, that option --start-time allows to set backup_id and restore""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore FULL backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_full'), + backup_id=base36enc(startTime)) + + #FULL backup with incorrect start time + try: + startTime = str(int(time()-100000)) + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + 'Expecting Error because start time for new backup must be newer ' + '\n Output: {0} \n CMD: {1}'.format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertRegex( + e.message, + r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + # DELTA backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore DELTA backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_delta'), + backup_id=base36enc(startTime)) + + # PAGE backup + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='page', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PAGE backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_page'), + backup_id=base36enc(startTime)) + + # PTRACK backup + if self.ptrack: + node.safe_psql( + 'postgres', + 'create extension ptrack') + + startTime = int(time()) + self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(str(startTime))]) + # restore PTRACK backup by backup_id calculated from start-time + self.restore_node( + backup_dir, 'node', + data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_ptrack'), + backup_id=base36enc(startTime)) + + # @unittest.skip("skip") + def test_start_time_few_nodes(self): + """Test, that we can synchronize backup_id's for different DBs""" + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir1 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup1') + self.init_pb(backup_dir1) + self.add_instance(backup_dir1, 'node1', node1) + self.set_archiving(backup_dir1, 'node1', node1) + node1.slow_start() + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + backup_dir2 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup2') + self.init_pb(backup_dir2) + self.add_instance(backup_dir2, 'node2', node2) + self.set_archiving(backup_dir2, 'node2', node2) + node2.slow_start() + + # FULL backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[0] + show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # DELTA backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='delta', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[1] + show_backup2 = self.show_pb(backup_dir2, 'node2')[1] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PAGE backup + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='page', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[2] + show_backup2 = self.show_pb(backup_dir2, 'node2')[2] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + # PTRACK backup + if self.ptrack: + node1.safe_psql( + 'postgres', + 'create extension ptrack') + node2.safe_psql( + 'postgres', + 'create extension ptrack') + + startTime = str(int(time())) + self.backup_node( + backup_dir1, 'node1', node1, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + self.backup_node( + backup_dir2, 'node2', node2, backup_type='ptrack', + options=['--stream', '--start-time={0}'.format(startTime)]) + show_backup1 = self.show_pb(backup_dir1, 'node1')[3] + show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + self.assertEqual(show_backup1['id'], show_backup2['id']) + + def test_regress_issue_585(self): + """https://github.com/postgrespro/pg_probackup/issues/585""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # create couple of files that looks like db files + with open(os.path.join(node.data_dir, 'pg_multixact/offsets/1000'),'wb') as f: + pass + with open(os.path.join(node.data_dir, 'pg_multixact/members/1000'),'wb') as f: + pass + + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream']) + + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream'], + return_id=False, + ) + self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') + + node.cleanup() + + output = self.restore_node(backup_dir, 'node', node) + self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') + + def test_2_delta_backups(self): + """https://github.com/postgrespro/pg_probackup/issues/596""" + node = self.make_simple_node('node', + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + full_backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + + # delta backup mode + delta_backup_id1 = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + + delta_backup_id2 = self.backup_node( + backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + + # postgresql.conf and pg_hba.conf shouldn't be copied + conf_file = os.path.join(backup_dir, 'backups', 'node', delta_backup_id1, 'database', 'postgresql.conf') + self.assertFalse( + os.path.exists(conf_file), + "File should not exist: {0}".format(conf_file)) + conf_file = os.path.join(backup_dir, 'backups', 'node', delta_backup_id2, 'database', 'postgresql.conf') + print(conf_file) + self.assertFalse( + os.path.exists(conf_file), + "File should not exist: {0}".format(conf_file)) diff --git a/tests/catchup_test.py b/tests/catchup_test.py new file mode 100644 index 000000000..cf8388dd2 --- /dev/null +++ b/tests/catchup_test.py @@ -0,0 +1,1626 @@ +import os +from pathlib import Path +import signal +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +class CatchupTest(ProbackupTest, unittest.TestCase): + +######################################### +# Basic tests +######################################### + def test_basic_full_catchup(self): + """ + Test 'multithreaded basebackup' mode (aka FULL catchup) + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do full catchup + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') + + def test_full_catchup_with_tablespace(self): + """ + Test tablespace transfers + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') + self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path) + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do full catchup with tablespace mapping + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # make changes in master tablespace + src_pg.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + src_pg.stop() + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + + def test_basic_delta_catchup(self): + """ + Test delta catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') + + def test_basic_ptrack_catchup(self): + """ + Test ptrack catchup + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') + + def test_tli_delta_catchup(self): + """ + Test that we correctly follow timeline change with delta catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: promote source + src_pg.stop() + self.set_replica(dst_pg, src_pg) # fake replication + src_pg.slow_start(replica = True) + src_pg.promote() + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do catchup (src_tli = 2, dst_tli = 1) + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + self.set_replica(master = src_pg, replica = dst_pg) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + dst_pg.stop() + + # do catchup (src_tli = 2, dst_tli = 2) + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Cleanup + src_pg.stop() + + def test_tli_ptrack_catchup(self): + """ + Test that we correctly follow timeline change with ptrack catchup + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: promote source + src_pg.stop() + self.set_replica(dst_pg, src_pg) # fake replication + src_pg.slow_start(replica = True) + src_pg.promote() + + src_pg.safe_psql("postgres", "CHECKPOINT") # force postgres to update tli in 'SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()' + src_tli = src_pg.safe_psql("postgres", "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()").decode('utf-8').rstrip() + self.assertEqual(src_tli, "2", "Postgres didn't update TLI after promote") + + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do catchup (src_tli = 2, dst_tli = 1) + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + self.set_replica(master = src_pg, replica = dst_pg) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + dst_pg.stop() + + # do catchup (src_tli = 2, dst_tli = 2) + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Cleanup + src_pg.stop() + +######################################### +# Test various corner conditions +######################################### + def test_table_drop_with_delta(self): + """ + Test that dropped table in source will be dropped in delta catchup'ed instance too + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + # perform checkpoint twice to ensure, that datafile is actually deleted on filesystem + src_pg.safe_psql("postgres", "DROP TABLE ultimate_question") + src_pg.safe_psql("postgres", "CHECKPOINT") + src_pg.safe_psql("postgres", "CHECKPOINT") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_table_drop_with_ptrack(self): + """ + Test that dropped table in source will be dropped in ptrack catchup'ed instance too + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + # perform checkpoint twice to ensure, that datafile is actually deleted on filesystem + src_pg.safe_psql("postgres", "DROP TABLE ultimate_question") + src_pg.safe_psql("postgres", "CHECKPOINT") + src_pg.safe_psql("postgres", "CHECKPOINT") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_tablefile_truncation_with_delta(self): + """ + Test that truncated table in source will be truncated in delta catchup'ed instance too + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE SEQUENCE t_seq; " + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 1024) i") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dest_options = {} + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.safe_psql("postgres", "DELETE FROM t_heap WHERE ctid >= '(11,0)'") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_tablefile_truncation_with_ptrack(self): + """ + Test that truncated table in source will be truncated in ptrack catchup'ed instance too + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE SEQUENCE t_seq; " + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 1024) i") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dest_options = {} + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.safe_psql("postgres", "DELETE FROM t_heap WHERE ctid >= '(11,0)'") + src_pg.safe_psql("postgres", "VACUUM t_heap") + + # do ptrack catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # Check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + +######################################### +# Test reaction on user errors +######################################### + def test_local_tablespace_without_mapping(self): + """ + Test that we detect absence of needed --tablespace-mapping option + """ + if self.remote: + self.skipTest('Skipped because this test tests local catchup error handling') + + src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'src')) + src_pg.slow_start() + + tblspace_path = self.get_tblspace_path(src_pg, 'tblspace') + self.create_tblspace_in_node( + src_pg, 'tblspace', + tblspc_path = tblspace_path) + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace AS SELECT 42 AS answer") + + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + ] + ) + self.assertEqual(1, 0, "Expecting Error because '-T' parameter is not specified.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Local catchup executed, but source database contains tablespace', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + + def test_running_dest_postmaster(self): + """ + Test that we detect running postmaster in destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + # leave running destination postmaster + # so don't call dst_pg.stop() + + # try delta catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because postmaster in destination is running.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Postmaster with pid ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + + def test_same_db_id(self): + """ + Test that we detect different id's of source and destination + """ + # preparation: + # source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + # destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + # fake destination + fake_dst_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_dst')) + # fake source + fake_src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_src')) + + # try delta catchup (src (with correct src conn), fake_dst) + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = fake_dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Database identifiers mismatch: ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try delta catchup (fake_src (with wrong src conn), dst) + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = fake_src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Database identifiers mismatch: ', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + + def test_tli_destination_mismatch(self): + """ + Test that we detect TLI mismatch in destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + self.set_replica(src_pg, dst_pg) + dst_pg.slow_start(replica = True) + dst_pg.promote() + dst_pg.stop() + + # preparation 3: "useful" changes + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # try catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_query_result = dst_pg.table_checksum("ultimate_question") + dst_pg.stop() + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + except ProbackupException as e: + self.assertIn( + 'ERROR: Source is behind destination in timeline history', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + + def test_tli_source_mismatch(self): + """ + Test that we detect TLI mismatch in source history + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: fake source (promouted copy) + fake_src_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'fake_src')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = fake_src_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + fake_src_options = {} + fake_src_options['port'] = str(fake_src_pg.port) + self.set_auto_conf(fake_src_pg, fake_src_options) + self.set_replica(src_pg, fake_src_pg) + fake_src_pg.slow_start(replica = True) + fake_src_pg.promote() + self.switch_wal_segment(fake_src_pg) + fake_src_pg.safe_psql( + "postgres", + "CREATE TABLE t_heap AS SELECT i AS id, " + "md5(i::text) AS text, " + "md5(repeat(i::text, 10))::tsvector AS tsvector " + "FROM generate_series(0, 256) i") + self.switch_wal_segment(fake_src_pg) + fake_src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 'trash' AS garbage") + + # preparation 3: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + # preparation 4: "useful" changes + src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + + # try catchup + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = fake_src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(fake_src_pg.port), '--stream'] + ) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_query_result = dst_pg.table_checksum("ultimate_question") + dst_pg.stop() + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination is not in source timeline history', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # Cleanup + src_pg.stop() + fake_src_pg.stop() + +######################################### +# Test unclean destination +######################################### + def test_unclean_delta_catchup(self): + """ + Test that we correctly recover uncleanly shutdowned destination + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # try #1 + try: + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination directory contains "backup_label" file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try #2 + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") + dst_pg.kill() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + + def test_unclean_ptrack_catchup(self): + """ + Test that we correctly recover uncleanly shutdowned destination + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: destination + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # try #1 + try: + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Destination directory contains "backup_label" file', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # try #2 + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") + dst_pg.kill() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.table_checksum("ultimate_question") + + # do delta catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # run&recover catchup'ed instance + src_pg.stop() + self.set_replica(master = src_pg, replica = dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + dst_pg.stop() + +######################################### +# Test replication slot logic +# +# -S, --slot=SLOTNAME replication slot to use +# --temp-slot use temporary replication slot +# -P --perm-slot create permanent replication slot +# --primary-slot-name=SLOTNAME value for primary_slot_name parameter +# +# 1. if "--slot" is used - try to use already existing slot with given name +# 2. if "--slot" and "--perm-slot" used - try to create permanent slot and use it. +# 3. If "--perm-slot " flag is used without "--slot" option - use generic slot name like "pg_probackup_perm_slot" +# 4. If "--perm-slot " flag is used and permanent slot already exists - fail with error. +# 5. "--perm-slot" and "--temp-slot" flags cannot be used together. +######################################### + def test_catchup_with_replication_slot(self): + """ + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # 1a. --slot option + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1a')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=nonexistentslot_1a' + ] + ) + self.assertEqual(1, 0, "Expecting Error because replication slot does not exist.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: replication slot "nonexistentslot_1a" does not exist', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # 1b. --slot option + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1b')) + src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_1b')") + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=existentslot_1b' + ] + ) + + # 2a. --slot --perm-slot + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2a')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=nonexistentslot_2a', + '--perm-slot' + ] + ) + + # 2b. and 4. --slot --perm-slot + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2b')) + src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_2b')") + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--slot=existentslot_2b', + '--perm-slot' + ] + ) + self.assertEqual(1, 0, "Expecting Error because replication slot already exist.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: replication slot "existentslot_2b" already exists', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + # 3. --perm-slot --slot + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_3')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot' + ] + ) + slot_name = src_pg.safe_psql( + "postgres", + "SELECT slot_name FROM pg_catalog.pg_replication_slots " + "WHERE slot_name NOT LIKE '%existentslot%' " + "AND slot_type = 'physical'" + ).decode('utf-8').rstrip() + self.assertEqual(slot_name, 'pg_probackup_perm_slot', 'Slot name mismatch') + + # 5. --perm-slot --temp-slot (PG>=10) + if self.get_version(src_pg) >= self.version_to_num('10.0'): + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_5')) + try: + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot' + ] + ) + self.assertEqual(1, 0, "Expecting Error because conflicting options --perm-slot and --temp-slot used together\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + + #self.assertEqual(1, 0, 'Stop test') + +######################################### +# --exclude-path +######################################### + def test_catchup_with_exclude_path(self): + """ + various syntetic tests for --exclude-path option + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # test 1 + os.mkdir(os.path.join(src_pg.data_dir, 'src_usefull_dir')) + with open(os.path.join(os.path.join(src_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')), 'w') as f: + f.write('garbage') + f.flush() + f.close + os.mkdir(os.path.join(src_pg.data_dir, 'src_garbage_dir')) + with open(os.path.join(os.path.join(src_pg.data_dir, 'src_garbage_dir', 'src_garbage_file')), 'w') as f: + f.write('garbage') + f.flush() + f.close + + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}'.format(os.path.join(src_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')), + '-x', '{0}'.format(os.path.join(src_pg.data_dir, 'src_garbage_dir')), + ] + ) + + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_garbage_dir')).exists()) + + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # test 2 + os.mkdir(os.path.join(dst_pg.data_dir, 'dst_garbage_dir')) + os.mkdir(os.path.join(dst_pg.data_dir, 'dst_usefull_dir')) + with open(os.path.join(os.path.join(dst_pg.data_dir, 'dst_usefull_dir', 'dst_usefull_file')), 'w') as f: + f.write('gems') + f.flush() + f.close + + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path=src_usefull_dir/src_garbage_file', + '--exclude-path=src_garbage_dir', + '--exclude-path={0}'.format(os.path.join(dst_pg.data_dir, 'dst_usefull_dir')), + ] + ) + + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_usefull_dir', 'src_garbage_file')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'src_garbage_dir')).exists()) + self.assertFalse(Path(os.path.join(dst_pg.data_dir, 'dst_garbage_dir')).exists()) + self.assertTrue(Path(os.path.join(dst_pg.data_dir, 'dst_usefull_dir', 'dst_usefull_file')).exists()) + + #self.assertEqual(1, 0, 'Stop test') + src_pg.stop() + + def test_config_exclusion(self): + """ + Test that catchup can preserve dest replication config + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question(answer int)") + + # preparation 2: make lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg._assign_master(src_pg) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 1: do delta catchup with relative exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path=postgresql.conf', + '--exclude-path=postgresql.auto.conf', + '--exclude-path=recovery.conf', + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(42)") + src_query_result = src_pg.table_checksum("ultimate_question") + dst_pg.catchup() # wait for replication + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # preparation 4: make changes on master (source) + dst_pg.stop() + #src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 2: do delta catchup with absolute source exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}/postgresql.conf'.format(src_pg.data_dir), + '--exclude-path={0}/postgresql.auto.conf'.format(src_pg.data_dir), + '--exclude-path={0}/recovery.conf'.format(src_pg.data_dir), + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(2*42)") + src_query_result = src_pg.table_checksum("ultimate_question") + dst_pg.catchup() # wait for replication + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # preparation 5: make changes on master (source) + dst_pg.stop() + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # test 3: do delta catchup with absolute destination exclusion paths + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--exclude-path={0}/postgresql.conf'.format(dst_pg.data_dir), + '--exclude-path={0}/postgresql.auto.conf'.format(dst_pg.data_dir), + '--exclude-path={0}/recovery.conf'.format(dst_pg.data_dir), + ] + ) + + # run&recover catchup'ed instance + # don't set destination db port and recover options + dst_pg.slow_start(replica = True) + + # check: run verification query + src_pg.safe_psql("postgres", "INSERT INTO ultimate_question VALUES(3*42)") + src_query_result = src_pg.table_checksum("ultimate_question") + dst_pg.catchup() # wait for replication + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # Cleanup + src_pg.stop() + dst_pg.stop() + #self.assertEqual(1, 0, 'Stop test') + +######################################### +# --dry-run +######################################### + def test_dry_run_catchup_full(self): + """ + Test dry-run option for full catchup + """ + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do full catchup + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run'] + ) + + # compare data dirs before and after catchup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_dry_run_catchup_ptrack(self): + """ + Test dry-run option for catchup in incremental ptrack mode + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + ptrack_enable = True, + initdb_params = ['--data-checksums'] + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do incremental catchup + self.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run'] + ) + + # compare data dirs before and after cathup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_dry_run_catchup_delta(self): + """ + Test dry-run option for catchup in incremental delta mode + """ + + # preparation 1: source + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True, + initdb_params = ['--data-checksums'], + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do delta catchup + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', "--dry-run"] + ) + + # compare data dirs before and after cathup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + + # Cleanup + src_pg.stop() + + def test_pgdata_is_ignored(self): + """ In catchup we still allow PGDATA to be set either from command line + or from the env var. This test that PGDATA is actually ignored and + --source-pgadta is used instead + """ + node = self.make_simple_node('node', + set_replication = True + ) + node.slow_start() + + # do full catchup + dest = self.make_empty_node('dst') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = node.data_dir, + destination_node = dest, + options = ['-d', 'postgres', '-p', str(node.port), '--stream', '--pgdata=xxx'] + ) + + self.compare_pgdata( + self.pgdata_content(node.data_dir), + self.pgdata_content(dest.data_dir) + ) + + os.environ['PGDATA']='xxx' + + dest2 = self.make_empty_node('dst') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = node.data_dir, + destination_node = dest2, + options = ['-d', 'postgres', '-p', str(node.port), '--stream'] + ) + + self.compare_pgdata( + self.pgdata_content(node.data_dir), + self.pgdata_content(dest2.data_dir) + ) diff --git a/tests/cfs_backup.py b/tests/cfs_backup_test.py similarity index 89% rename from tests/cfs_backup.py rename to tests/cfs_backup_test.py index 5a3665518..fb4a6c6b8 100644 --- a/tests/cfs_backup.py +++ b/tests/cfs_backup_test.py @@ -6,7 +6,6 @@ from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_backup' tblspace_name = 'cfs_tblspace' @@ -14,15 +13,14 @@ class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): # --- Begin --- # @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def setUp(self): - self.fname = self.id().split('.')[3] self.backup_dir = os.path.join( - self.tmp_path, module_name, self.fname, 'backup') + self.tmp_path, self.module_name, self.fname, 'backup') self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, self.fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, + ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'ptrack_enable': 'on', 'cfs_encryption': 'off', 'max_wal_senders': '2', 'shared_buffers': '200MB' @@ -35,18 +33,26 @@ def setUp(self): self.node.slow_start() + self.node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name) - ) - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, + tblspace_name)) + + self.assertIn( + tblspace_name, str(tblspace), "ERROR: The tablespace not created " - "or it create without compressions" - ) + "or it create without compressions") + + self.assertIn( + "compression=true", str(tblspace), + "ERROR: The tablespace not created " + "or it create without compressions") self.assertTrue( find_by_name( @@ -163,12 +169,18 @@ def test_fullbackup_after_create_table(self): "ERROR: File pg_compression not found in {0}".format( os.path.join(self.backup_dir, 'node', backup_id)) ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) + + # check cfm size + cfms = find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) # @unittest.expectedFailure # @unittest.skip("skip") @@ -405,6 +417,55 @@ def test_fullbackup_empty_tablespace_page_after_create_table(self): "ERROR: .cfm files not found in backup dir" ) + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_page_doesnt_store_unchanged_cfm(self): + """ + Case: Test page backup doesn't store cfm file if table were not modified + """ + + self.node.safe_psql( + "postgres", + "CREATE TABLE {0} TABLESPACE {1} " + "AS SELECT i AS id, MD5(i::text) AS text, " + "MD5(repeat(i::text,10))::tsvector AS tsvector " + "FROM generate_series(0,256) i".format('t1', tblspace_name) + ) + + self.node.safe_psql("postgres", "checkpoint") + + backup_id_full = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='full') + + self.assertTrue( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], + ['.cfm']), + "ERROR: .cfm files not found in backup dir" + ) + + backup_id = self.backup_node( + self.backup_dir, 'node', self.node, backup_type='page') + + show_backup = self.show_pb(self.backup_dir, 'node', backup_id) + self.assertEqual( + "OK", + show_backup["status"], + "ERROR: Incremental backup status is not valid. \n " + "Current backup status={0}".format(show_backup["status"]) + ) + self.assertTrue( + find_by_name( + [self.get_tblspace_path(self.node, tblspace_name)], + ['pg_compression']), + "ERROR: File pg_compression not found" + ) + self.assertFalse( + find_by_extensions( + [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], + ['.cfm']), + "ERROR: .cfm files is found in backup dir" + ) + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') @@ -473,7 +534,7 @@ def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): ) # --- Section: Incremental from fill tablespace --- # - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table(self): @@ -537,7 +598,7 @@ def test_fullbackup_after_create_table_ptrack_after_create_table(self): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): @@ -603,7 +664,7 @@ def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): ) ) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table(self): @@ -686,7 +747,7 @@ def test_multiple_segments(self): 't_heap', tblspace_name) ) - full_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = self.node.table_checksum("t_heap") try: backup_id_full = self.backup_node( @@ -708,7 +769,7 @@ def test_multiple_segments(self): 't_heap') ) - page_result = self.node.safe_psql("postgres", "SELECT * FROM t_heap") + page_result = self.node.table_checksum("t_heap") try: backup_id_page = self.backup_node( @@ -738,16 +799,18 @@ def test_multiple_segments(self): # CHECK FULL BACKUP self.node.stop() self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) + shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_full, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( full_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + self.node.table_checksum("t_heap"), 'Lost data after restore') # CHECK PAGE BACKUP @@ -757,12 +820,16 @@ def test_multiple_segments(self): self.get_tblspace_path(self.node, tblspace_name), ignore_errors=True) self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + self.backup_dir, 'node', self.node, backup_id=backup_id_page, + options=[ + "-j", "4", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) + self.node.slow_start() self.assertEqual( page_result, - self.node.safe_psql("postgres", "SELECT * FROM t_heap"), + self.node.table_checksum("t_heap"), 'Lost data after restore') # @unittest.expectedFailure @@ -786,8 +853,7 @@ def test_multiple_segments_in_multiple_tablespaces(self): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1) - ) + 't_heap_1', tblspace_name_1)) self.node.safe_psql( "postgres", @@ -795,13 +861,10 @@ def test_multiple_segments_in_multiple_tablespaces(self): "AS SELECT i AS id, MD5(i::text) AS text, " "MD5(repeat(i::text,10))::tsvector AS tsvector " "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2) - ) + 't_heap_2', tblspace_name_2)) - full_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - full_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") + full_result_1 = self.node.table_checksum("t_heap_1") + full_result_2 = self.node.table_checksum("t_heap_2") try: backup_id_full = self.backup_node( @@ -832,10 +895,8 @@ def test_multiple_segments_in_multiple_tablespaces(self): 't_heap_2') ) - page_result_1 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_1") - page_result_2 = self.node.safe_psql( - "postgres", "SELECT * FROM t_heap_2") + page_result_1 = self.node.table_checksum("t_heap_1") + page_result_2 = self.node.table_checksum("t_heap_2") try: backup_id_page = self.backup_node( @@ -864,57 +925,47 @@ def test_multiple_segments_in_multiple_tablespaces(self): # CHECK FULL BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_full, options=["-j", "4"]) + backup_id=backup_id_full, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( full_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + self.node.table_checksum("t_heap_1"), 'Lost data after restore') self.assertEqual( full_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + self.node.table_checksum("t_heap_2"), 'Lost data after restore') # CHECK PAGE BACKUP self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_1), - ignore_errors=True) - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name_2), - ignore_errors=True) self.restore_node( self.backup_dir, 'node', self.node, - backup_id=backup_id_page, options=["-j", "4"]) + backup_id=backup_id_page, + options=[ + "-j", "4", "--incremental-mode=checksum", + "--recovery-target=immediate", + "--recovery-target-action=promote"]) self.node.slow_start() + self.assertEqual( page_result_1, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_1"), + self.node.table_checksum("t_heap_1"), 'Lost data after restore') self.assertEqual( page_result_2, - self.node.safe_psql("postgres", "SELECT * FROM t_heap_2"), + self.node.table_checksum("t_heap_2"), 'Lost data after restore') - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_fullbackup_after_create_table_page_after_create_table_stream(self): @@ -981,7 +1032,6 @@ def test_fullbackup_after_create_table_page_after_create_table_stream(self): ) # --- Make backup with not valid data(broken .cfm) --- # - @unittest.expectedFailure # @unittest.skip("skip") @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_delete_random_cfm_file_from_tablespace_dir(self): @@ -993,6 +1043,11 @@ def test_delete_random_cfm_file_from_tablespace_dir(self): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_cmf = find_by_extensions( [self.get_tblspace_path(self.node, tblspace_name)], ['.cfm']) @@ -1042,6 +1097,11 @@ def test_delete_random_data_file_from_tablespace_dir(self): "FROM generate_series(0,256) i".format('t1', tblspace_name) ) + self.node.safe_psql( + "postgres", + "CHECKPOINT" + ) + list_data_files = find_by_pattern( [self.get_tblspace_path(self.node, tblspace_name)], '^.*/\d+$') @@ -1147,10 +1207,6 @@ def test_broken_file_pg_compression_into_tablespace_dir(self): ) # # --- End ---# -# @unittest.skipUnless(ProbackupTest.enterprise, 'skip') -# def tearDown(self): -# self.node.cleanup() -# self.del_test_dir(module_name, self.fname) #class CfsBackupEncTest(CfsBackupNoEncTest): diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py new file mode 100644 index 000000000..f6760b72c --- /dev/null +++ b/tests/cfs_catchup_test.py @@ -0,0 +1,117 @@ +import os +import unittest +import random +import shutil + +from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + + +class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): + + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') + def test_full_catchup_with_tablespace(self): + """ + Test tablespace transfers + """ + # preparation + src_pg = self.make_simple_node( + base_dir = os.path.join(self.module_name, self.fname, 'src'), + set_replication = True + ) + src_pg.slow_start() + tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') + self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") + src_query_result = src_pg.table_checksum("ultimate_question") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # do full catchup with tablespace mapping + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') + self.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # check cfm size + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + + # make changes in master tablespace + src_pg.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + src_pg.safe_psql( + "postgres", + "CHECKPOINT") + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') + + # and now delta backup + dst_pg.stop() + + self.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream', + '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) + ] + ) + + # check cfm size again + cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) + self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") + for cfm in cfms: + size = os.stat(cfm).st_size + self.assertLessEqual(size, 4096, + "ERROR: {0} is not truncated (has size {1} > 4096)".format( + cfm, size + )) + + # run&recover catchup'ed instance + dst_options = {} + dst_options['port'] = str(dst_pg.port) + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + + + # 3rd check: run verification query + src_query_result = src_pg.table_checksum("ultimate_question") + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore.py b/tests/cfs_restore_test.py similarity index 91% rename from tests/cfs_restore.py rename to tests/cfs_restore_test.py index 07cf891aa..2fa35e71a 100644 --- a/tests/cfs_restore.py +++ b/tests/cfs_restore_test.py @@ -15,20 +15,17 @@ from .helpers.cfs_helpers import find_by_name from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -module_name = 'cfs_restore' - tblspace_name = 'cfs_tblspace' tblspace_name_new = 'cfs_tblspace_new' class CfsRestoreBase(ProbackupTest, unittest.TestCase): + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def setUp(self): - self.fname = self.id().split('.')[3] - self.backup_dir = os.path.join(self.tmp_path, module_name, self.fname, 'backup') + self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, self.fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -60,14 +57,11 @@ def setUp(self): def add_data_in_cluster(self): pass - def tearDown(self): - self.node.cleanup() - self.del_test_dir(module_name, self.fname) - class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_empty_tablespace_from_fullbackup(self): """ Case: Restore empty tablespace from valid full backup. @@ -102,7 +96,7 @@ def test_restore_empty_tablespace_from_fullbackup(self): tblspace = self.node.safe_psql( "postgres", "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ) + ).decode("UTF-8") self.assertTrue( tblspace_name in tblspace and "compression=true" in tblspace, "ERROR: The tablespace not restored or it restored without compressions" @@ -118,14 +112,12 @@ def add_data_in_cluster(self): MD5(repeat(i::text,10))::tsvector AS tsvector \ FROM generate_series(0,1e5) i'.format('t1', tblspace_name) ) - self.table_t1 = self.node.safe_psql( - "postgres", - "SELECT * FROM t1" - ) + self.table_t1 = self.node.table_checksum("t1") # --- Restore from full backup ---# # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location(self): """ Case: Restore instance from valid full backup to old location. @@ -159,12 +151,13 @@ def test_restore_from_fullbackup_to_old_location(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_3_jobs(self): """ Case: Restore instance from valid full backup to old location. @@ -197,12 +190,13 @@ def test_restore_from_fullbackup_to_old_location_3_jobs(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_new_location(self): """ Case: Restore instance from valid full backup to new location. @@ -211,7 +205,7 @@ def test_restore_from_fullbackup_to_new_location(self): self.node.cleanup() shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) node_new.cleanup() try: @@ -239,13 +233,14 @@ def test_restore_from_fullbackup_to_new_location(self): ) self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + node_new.table_checksum("t1"), + self.table_t1 ) node_new.cleanup() # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_new_location_5_jobs(self): """ Case: Restore instance from valid full backup to new location. @@ -254,7 +249,7 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): self.node.cleanup() shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(module_name, self.fname)) + node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) node_new.cleanup() try: @@ -282,13 +277,14 @@ def test_restore_from_fullbackup_to_new_location_5_jobs(self): ) self.assertEqual( - repr(node_new.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + node_new.table_checksum("t1"), + self.table_t1 ) node_new.cleanup() # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): self.node.stop() self.node.cleanup() @@ -329,12 +325,13 @@ def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure # @unittest.skip("skip") + @unittest.skipUnless(ProbackupTest.enterprise, 'skip') def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): self.node.stop() self.node.cleanup() @@ -375,8 +372,8 @@ def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs( ) self.assertEqual( - repr(self.node.safe_psql("postgres", "SELECT * FROM %s" % 't1')), - repr(self.table_t1) + self.node.table_checksum("t1"), + self.table_t1 ) # @unittest.expectedFailure diff --git a/tests/cfs_validate_backup.py b/tests/cfs_validate_backup_test.py similarity index 94% rename from tests/cfs_validate_backup.py rename to tests/cfs_validate_backup_test.py index eea6f0e21..343020dfc 100644 --- a/tests/cfs_validate_backup.py +++ b/tests/cfs_validate_backup_test.py @@ -5,7 +5,6 @@ from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'cfs_validate_backup' tblspace_name = 'cfs_tblspace' diff --git a/tests/checkdb.py b/tests/checkdb_test.py similarity index 64% rename from tests/checkdb.py rename to tests/checkdb_test.py index 6c25293ab..eb46aea19 100644 --- a/tests/checkdb.py +++ b/tests/checkdb_test.py @@ -9,18 +9,16 @@ import time -module_name = 'checkdb' - - class CheckdbTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_checkdb_amcheck_only_sanity(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) @@ -36,6 +34,17 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") + + node.safe_psql( + "postgres", + "create table idxpart (a int) " + "partition by range (a)") + + # there aren't partitioned indexes on 10 and lesser versions + if self.get_version(node) >= 110000: + node.safe_psql( + "postgres", + "create index on idxpart(a)") try: node.safe_psql( @@ -122,7 +131,7 @@ def test_checkdb_amcheck_only_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: required parameter not specified: --instance", + "ERROR: Required parameter not specified: --instance", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -210,15 +219,15 @@ def test_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() + node.stop() # @unittest.skip("skip") def test_basic_checkdb_amcheck_only_sanity(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), set_replication=True, initdb_params=['--data-checksums']) @@ -259,13 +268,13 @@ def test_basic_checkdb_amcheck_only_sanity(self): node.data_dir, node.safe_psql( "db1", - "select pg_relation_filepath('pgbench_accounts_pkey')").rstrip()) + "select pg_relation_filepath('pgbench_accounts_pkey')").decode('utf-8').rstrip()) index_path_2 = os.path.join( node.data_dir, node.safe_psql( "db2", - "select pg_relation_filepath('some_index')").rstrip()) + "select pg_relation_filepath('some_index')").decode('utf-8').rstrip()) try: self.checkdb_node( @@ -348,18 +357,17 @@ def test_basic_checkdb_amcheck_only_sanity(self): log_file_content) # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + node.stop() # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): """make node, corrupt some pages, check that checkdb failed""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -376,7 +384,7 @@ def test_checkdb_block_validation_sanity(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # sanity try: @@ -389,7 +397,7 @@ def test_checkdb_block_validation_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: required parameter not specified: PGDATA (-D, --pgdata)", + "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -444,15 +452,105 @@ def test_checkdb_block_validation_sanity(self): e.message) # Clean after yourself - self.del_test_dir(module_name, fname) + node.stop() + + def test_checkdb_checkunique(self): + """Test checkunique parameter of amcheck.bt_index_check function""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + node.slow_start() + + try: + node.safe_psql( + "postgres", + "create extension amcheck") + except QueryException as e: + node.safe_psql( + "postgres", + "create extension amcheck_next") + + # Part of https://commitfest.postgresql.org/32/2976/ patch test + node.safe_psql( + "postgres", + "CREATE TABLE bttest_unique(a varchar(50), b varchar(1500), c bytea, d varchar(50)); " + "ALTER TABLE bttest_unique SET (autovacuum_enabled = false); " + "CREATE UNIQUE INDEX bttest_unique_idx ON bttest_unique(a,b); " + "UPDATE pg_catalog.pg_index SET indisunique = false " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "INSERT INTO bttest_unique " + " SELECT i::text::varchar, " + " array_to_string(array( " + " SELECT substr('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', ((random()*(36-1)+1)::integer), 1) " + " FROM generate_series(1,1300)),'')::varchar, " + " i::text::bytea, i::text::varchar " + " FROM generate_series(0,1) AS i, generate_series(0,30) AS x; " + "UPDATE pg_catalog.pg_index SET indisunique = true " + "WHERE indrelid = (SELECT oid FROM pg_catalog.pg_class WHERE relname = 'bttest_unique'); " + "DELETE FROM bttest_unique WHERE ctid::text='(0,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,2)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(4,3)'; " + "DELETE FROM bttest_unique WHERE ctid::text='(9,3)';") + + # run without checkunique option (error will not detected) + output = self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '-d', 'postgres', '-p', str(node.port)]) + + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + output) + self.assertIn( + 'All checked indexes are valid', + output) + + # run with checkunique option + try: + self.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--checkunique', + '-d', 'postgres', '-p', str(node.port)]) + if (ProbackupTest.enterprise and + (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400)): + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of index corruption\n" + " Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + else: + self.assertRegex( + self.output, + r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") + except ProbackupException as e: + self.assertIn( + "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.", + e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertIn( + "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", + e.message) + + # Clean after yourself + node.stop() # @unittest.skip("skip") def test_checkdb_sigint_handling(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -473,14 +571,15 @@ def test_checkdb_sigint_handling(self): gdb = self.checkdb_node( backup_dir, 'node', gdb=True, options=[ - '-d', 'postgres', '-j', '4', + '-d', 'postgres', '-j', '2', '--skip-block-validation', + '--progress', '--amcheck', '-p', str(node.port)]) gdb.set_breakpoint('amcheck_one_index') gdb.run_until_break() - gdb.continue_execution_until_break(10) + gdb.continue_execution_until_break(20) gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') @@ -494,15 +593,15 @@ def test_checkdb_sigint_handling(self): self.assertNotIn('connection to client lost', output) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() + node.stop() # @unittest.skip("skip") def test_checkdb_with_least_privileges(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -554,16 +653,18 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - ) + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function + # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -580,6 +681,38 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' +# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # PG 10 + elif self.get_version(node) > 100000 and self.get_version(node) < 110000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' @@ -587,10 +720,59 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') + + if ProbackupTest.enterprise: + # amcheck-1.1 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') + else: + # amcheck-1.0 + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') + # >= 11 < 14 + elif self.get_version(node) > 110000 and self.get_version(node) < 140000: + node.safe_psql( + 'backupdb', + 'CREATE ROLE backup WITH LOGIN; ' + 'GRANT CONNECT ON DATABASE backupdb to backup; ' + 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' + 'GRANT USAGE ON SCHEMA public TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' + 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - ) - # >= 10 + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # checkunique parameter + if ProbackupTest.enterprise: + if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 + or self.get_version(node) >= 120800 and self.get_version(node) < 130000 + or self.get_version(node) >= 130400): + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + # >= 14 else: node.safe_psql( 'backupdb', @@ -606,6 +788,8 @@ def test_checkdb_with_least_privileges(self): 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' @@ -613,18 +797,22 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' - ) - -# if ProbackupTest.enterprise: -# node.safe_psql( -# "backupdb", -# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") -# -# node.safe_psql( -# "backupdb", -# "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') + + # checkunique parameter + if ProbackupTest.enterprise: + node.safe_psql( + "backupdb", + "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") + + if ProbackupTest.pgpro: + node.safe_psql( + 'backupdb', + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') # checkdb try: @@ -658,3 +846,6 @@ def test_checkdb_with_least_privileges(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) + + # Clean after yourself + node.stop() diff --git a/tests/compatibility.py b/tests/compatibility_test.py similarity index 74% rename from tests/compatibility.py rename to tests/compatibility_test.py index d2db2be28..7ae8baf9f 100644 --- a/tests/compatibility.py +++ b/tests/compatibility_test.py @@ -5,23 +5,99 @@ from sys import exit import shutil -module_name = 'compatibility' + +def check_manual_tests_enabled(): + return 'PGPROBACKUP_MANUAL' in os.environ and os.environ['PGPROBACKUP_MANUAL'] == 'ON' + + +def check_ssh_agent_path_exists(): + return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ + + +class CrossCompatibilityTest(ProbackupTest, unittest.TestCase): + @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') + @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') + # @unittest.skip("skip") + def test_catchup_with_different_remote_major_pg(self): + """ + Decription in jira issue PBCKP-236 + This test exposures ticket error using pg_probackup builds for both PGPROEE11 and PGPROEE9_6 + + Prerequisites: + - pg_probackup git tag for PBCKP 2.5.1 + - master pg_probackup build should be made for PGPROEE11 + - agent pg_probackup build should be made for PGPROEE9_6 + + Calling probackup PGPROEE9_6 pg_probackup agent from PGPROEE11 pg_probackup master for DELTA backup causes + the PBCKP-236 problem + + Please give env variables PROBACKUP_MANUAL=ON;PGPROBACKUP_SSH_AGENT_PATH= + for the test + + Please make path for agent's pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.ee.9.6/bin/' + without pg_probackup executable + """ + + self.verbose = True + self.remote = True + # please use your own local path like + # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' + pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] + + src_pg = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'src'), + set_replication=True, + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + + # do full catchup + dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + self.catchup_node( + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + + dst_options = {'port': str(dst_pg.port)} + self.set_auto_conf(dst_pg, dst_options) + dst_pg.slow_start() + dst_pg.stop() + + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question2 AS SELECT 42 AS answer") + + # do delta catchup with remote pg_probackup agent with another postgres major version + # this DELTA backup should fail without PBCKP-236 patch. + self.catchup_node( + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + # here's substitution of --remoge-path pg_probackup agent compiled with another postgres version + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--remote-path=' + pgprobackup_ssh_agent_path] + ) class CompatibilityTest(ProbackupTest, unittest.TestCase): + def setUp(self): + super().setUp() + if not self.probackup_old_path: + self.skipTest('PGPROBACKUPBIN_OLD is not set') + # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -47,7 +123,7 @@ def test_backward_compatibility_page(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -144,20 +220,15 @@ def test_backward_compatibility_page(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_delta(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -183,7 +254,7 @@ def test_backward_compatibility_delta(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -279,26 +350,20 @@ def test_backward_compatibility_delta(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_ptrack(self): """Description in jira issue PGPRO-434""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -309,10 +374,9 @@ def test_backward_compatibility_ptrack(self): self.set_archiving(backup_dir, 'node', node, old_binary=True) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=10) @@ -329,7 +393,7 @@ def test_backward_compatibility_ptrack(self): # RESTORE old FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -396,21 +460,15 @@ def test_backward_compatibility_ptrack(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_compression(self): """Description in jira issue PGPRO-434""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -431,7 +489,7 @@ def test_backward_compatibility_compression(self): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -557,9 +615,6 @@ def test_backward_compatibility_compression(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge(self): @@ -567,14 +622,11 @@ def test_backward_compatibility_merge(self): Create node, take FULL and PAGE backups with old binary, merge them with new binary """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -603,7 +655,7 @@ def test_backward_compatibility_merge(self): # restore OLD FULL with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -614,9 +666,6 @@ def test_backward_compatibility_merge(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_1(self): @@ -625,13 +674,11 @@ def test_backward_compatibility_merge_1(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -675,11 +722,11 @@ def test_backward_compatibility_merge_1(self): # check that in-place is disabled self.assertIn( "WARNING: In-place merge is disabled " - "because of program versions mismatch", output) + "because of storage format incompatibility", output) # restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -687,9 +734,6 @@ def test_backward_compatibility_merge_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_2(self): @@ -698,13 +742,11 @@ def test_backward_compatibility_merge_2(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -719,7 +761,7 @@ def test_backward_compatibility_merge_2(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node(backup_dir, 'node', node, old_binary=True) @@ -810,9 +852,6 @@ def test_backward_compatibility_merge_2(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_3(self): @@ -821,13 +860,11 @@ def test_backward_compatibility_merge_3(self): merge them with new binary. old binary version =< 2.2.7 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -842,7 +879,7 @@ def test_backward_compatibility_merge_3(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node( @@ -934,9 +971,6 @@ def test_backward_compatibility_merge_3(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_merge_4(self): @@ -944,13 +978,15 @@ def test_backward_compatibility_merge_4(self): Start merge between minor version, crash and retry it. old binary version =< 2.4.0 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if self.version_to_num(self.old_probackup_version) > self.version_to_num('2.4.0'): + self.assertTrue( + False, 'You need pg_probackup old_binary =< 2.4.0 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -965,7 +1001,7 @@ def test_backward_compatibility_merge_4(self): 'VACUUM pgbench_accounts') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # FULL backup with OLD binary self.backup_node( @@ -1008,8 +1044,81 @@ def test_backward_compatibility_merge_4(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_backward_compatibility_merge_5(self): + """ + Create node, take FULL and PAGE backups with old binary, + merge them with new binary. + old binary version >= STORAGE_FORMAT_VERSION (2.4.4) + """ + if self.version_to_num(self.old_probackup_version) < self.version_to_num('2.4.4'): + self.assertTrue( + False, 'OLD pg_probackup binary must be >= 2.4.4 for this test') + + self.assertNotEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num(self.probackup_version)) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir, old_binary=True) + self.add_instance(backup_dir, 'node', node, old_binary=True) + + self.set_archiving(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + node.pgbench_init(scale=20) + + # FULL backup with OLD binary + self.backup_node(backup_dir, 'node', node, old_binary=True) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=["-c", "1", "-T", "10", "--no-vacuum"]) + pgbench.wait() + pgbench.stdout.close() + + # PAGE1 backup with OLD binary + self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts') + + node.safe_psql( + 'postgres', + 'VACUUM pgbench_accounts') + + # PAGE2 backup with OLD binary + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='page', old_binary=True) + + pgdata = self.pgdata_content(node.data_dir) + + # merge chain created by old binary with new binary + output = self.merge_backup(backup_dir, "node", backup_id) + + # check that in-place is disabled + self.assertNotIn( + "WARNING: In-place merge is disabled " + "because of storage format incompatibility", output) + + # restore merged backup + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + self.restore_node(backup_dir, 'node', node_restored) + + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_page_vacuum_truncate(self): @@ -1022,13 +1131,11 @@ def test_page_vacuum_truncate(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1074,7 +1181,7 @@ def test_page_vacuum_truncate(self): pgdata3 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1113,9 +1220,6 @@ def test_page_vacuum_truncate(self): node_restored.slow_start() node_restored.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate_compression(self): """ @@ -1127,13 +1231,11 @@ def test_page_vacuum_truncate_compression(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1181,7 +1283,7 @@ def test_page_vacuum_truncate_compression(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1193,9 +1295,6 @@ def test_page_vacuum_truncate_compression(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_vacuum_truncate_compressed_1(self): """ @@ -1207,13 +1306,11 @@ def test_page_vacuum_truncate_compressed_1(self): and check data correctness old binary should be 2.2.x version """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1263,7 +1360,7 @@ def test_page_vacuum_truncate_compressed_1(self): pgdata3 = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1302,9 +1399,6 @@ def test_page_vacuum_truncate_compressed_1(self): node_restored.slow_start() node_restored.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_hidden_files(self): """ @@ -1313,13 +1407,11 @@ def test_hidden_files(self): with old binary, then try to delete backup with new binary """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -1332,5 +1424,80 @@ def test_hidden_files(self): self.delete_pb(backup_dir, 'node', backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_compatibility_tablespace(self): + """ + https://github.com/postgrespro/pg_probackup/issues/348 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node, old_binary=True) + node.slow_start() + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type="full", + options=["-j", "4", "--stream"], old_binary=True) + + tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old') + + self.create_tblspace_in_node( + node, 'tblspace', + tblspc_path=tblspace_old_path) + + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace tblspace " + "as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + + tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because tablespace mapping is incorrect" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Backup {0} has no tablespaceses, ' + 'nothing to remap'.format(backup_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.backup_node( + backup_dir, 'node', node, backup_type="delta", + options=["-j", "4", "--stream"], old_binary=True) + + self.restore_node( + backup_dir, 'node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)]) + + if self.paranoia: + pgdata = self.pgdata_content(node.data_dir) + + if self.paranoia: + pgdata_restored = self.pgdata_content(node_restored.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/compression.py b/tests/compression_test.py similarity index 83% rename from tests/compression.py rename to tests/compression_test.py index 321461d6e..55924b9d2 100644 --- a/tests/compression.py +++ b/tests/compression_test.py @@ -5,9 +5,6 @@ import subprocess -module_name = 'compression' - - class CompressionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -18,10 +15,9 @@ def test_basic_compression_stream_zlib(self): check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -36,7 +32,7 @@ def test_basic_compression_stream_zlib(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=[ @@ -49,7 +45,7 @@ def test_basic_compression_stream_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=[ @@ -61,7 +57,7 @@ def test_basic_compression_stream_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=zlib']) @@ -81,7 +77,7 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -97,7 +93,7 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -113,22 +109,18 @@ def test_basic_compression_stream_zlib(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - def test_compression_archive_zlib(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -142,7 +134,7 @@ def test_compression_archive_zlib(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=["--compress-algorithm=zlib"]) @@ -153,7 +145,7 @@ def test_compression_archive_zlib(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(0,2) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=["--compress-algorithm=zlib"]) @@ -163,7 +155,7 @@ def test_compression_archive_zlib(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=zlib']) @@ -183,7 +175,7 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -199,7 +191,7 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -215,23 +207,19 @@ def test_compression_archive_zlib(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_stream_pglz(self): """ make archive node, make full and page stream backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -246,7 +234,7 @@ def test_compression_stream_pglz(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream', '--compress-algorithm=pglz']) @@ -257,7 +245,7 @@ def test_compression_stream_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--stream', '--compress-algorithm=pglz']) @@ -268,7 +256,7 @@ def test_compression_stream_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream', '--compress-algorithm=pglz']) @@ -288,7 +276,7 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -304,7 +292,7 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -320,23 +308,19 @@ def test_compression_stream_pglz(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_archive_pglz(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -351,7 +335,7 @@ def test_compression_archive_pglz(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--compress-algorithm=pglz']) @@ -362,7 +346,7 @@ def test_compression_archive_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--compress-algorithm=pglz']) @@ -373,7 +357,7 @@ def test_compression_archive_pglz(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(200,300) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--compress-algorithm=pglz']) @@ -393,7 +377,7 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -409,7 +393,7 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() @@ -425,23 +409,19 @@ def test_compression_archive_pglz(self): repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_compression_wrong_algorithm(self): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -463,13 +443,10 @@ def test_compression_wrong_algorithm(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: invalid compress algorithm value "bla-blah"\n', + 'ERROR: Invalid compress algorithm value "bla-blah"\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incompressible_pages(self): """ @@ -477,10 +454,9 @@ def test_incompressible_pages(self): take backup with compression, make sure that page was not compressed, restore backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -517,6 +493,3 @@ def test_incompressible_pages(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/config.py b/tests/config_test.py similarity index 90% rename from tests/config.py rename to tests/config_test.py index b41382204..b1a0f9295 100644 --- a/tests/config.py +++ b/tests/config_test.py @@ -5,19 +5,16 @@ from sys import exit from shutil import copyfile -module_name = 'config' - class ConfigTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure # @unittest.skip("skip") def test_remove_instance_config(self): - """remove pg_probackup.conf""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """remove pg_probackup.conself.f""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -57,10 +54,9 @@ def test_remove_instance_config(self): # @unittest.skip("skip") def test_corrupt_backup_content(self): """corrupt backup_content.control""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) diff --git a/tests/delete.py b/tests/delete_test.py similarity index 88% rename from tests/delete.py rename to tests/delete_test.py index 8ebd7d13a..10100887d 100644 --- a/tests/delete.py +++ b/tests/delete_test.py @@ -2,10 +2,6 @@ import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess -from sys import exit - - -module_name = 'delete' class DeleteTest(ProbackupTest, unittest.TestCase): @@ -14,12 +10,11 @@ class DeleteTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_delete_full_backups(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -51,19 +46,15 @@ def test_delete_full_backups(self): self.assertEqual(show_backups[0]['id'], id_1) self.assertEqual(show_backups[1]['id'], id_3) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_del_instance_archive(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -83,19 +74,15 @@ def test_del_instance_archive(self): # Delete instance self.del_instance(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_archive_mix_compress_and_non_compressed_segments(self): """delete full backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), + base_dir="{0}/{1}/node".format(self.module_name, self.fname), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving( @@ -142,18 +129,14 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): '--retention-redundancy=3', '--delete-expired']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_increment_page(self): """delete increment and all after him""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -182,31 +165,26 @@ def test_delete_increment_page(self): self.assertEqual(show_backups[1]['backup-mode'], "FULL") self.assertEqual(show_backups[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_increment_ptrack(self): """delete increment and all after him""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), ptrack_enable=self.ptrack, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - 'postgres', - 'CREATE EXTENSION ptrack') + node.safe_psql( + 'postgres', + 'CREATE EXTENSION ptrack') # full backup mode self.backup_node(backup_dir, 'node', node) @@ -231,9 +209,6 @@ def test_delete_increment_ptrack(self): self.assertEqual(show_backups[1]['backup-mode'], "FULL") self.assertEqual(show_backups[1]['status'], "OK") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_orphaned_wal_segments(self): """ @@ -241,12 +216,11 @@ def test_delete_orphaned_wal_segments(self): delete second backup without --wal option, then delete orphaned wals via --wal option """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -303,9 +277,6 @@ def test_delete_orphaned_wal_segments(self): wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] self.assertEqual (0, len(wals), "Number of wals should be equal to 0") - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_wal_between_multiple_timelines(self): """ @@ -316,12 +287,11 @@ def test_delete_wal_between_multiple_timelines(self): [A1, B1) are deleted and backups B1 and A2 keep their WAL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -333,7 +303,7 @@ def test_delete_wal_between_multiple_timelines(self): node.pgbench_init(scale=3) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2) @@ -357,22 +327,18 @@ def test_delete_wal_between_multiple_timelines(self): self.validate_pb(backup_dir) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_backup_with_empty_control_file(self): """ take backup, truncate its control file, try to delete it via 'delete' command """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -398,18 +364,14 @@ def test_delete_backup_with_empty_control_file(self): self.delete_pb(backup_dir, 'node', backup_id=backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -522,9 +484,6 @@ def test_delete_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_multiple_descendants(self): """ @@ -537,12 +496,11 @@ def test_delete_multiple_descendants(self): FULLb | FULLa should be deleted """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -694,9 +652,6 @@ def test_delete_multiple_descendants(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delete_multiple_descendants_dry_run(self): """ @@ -707,12 +662,11 @@ def test_delete_multiple_descendants_dry_run(self): | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -799,17 +753,13 @@ def test_delete_multiple_descendants_dry_run(self): self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delete_error_backups(self): """delete increment and all after him""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -870,6 +820,3 @@ def test_delete_error_backups(self): self.assertEqual(show_backups[1]['status'], "OK") self.assertEqual(show_backups[2]['status'], "OK") self.assertEqual(show_backups[3]['status'], "OK") - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/delta.py b/tests/delta_test.py similarity index 84% rename from tests/delta.py rename to tests/delta_test.py index 0abdd1c2c..8736a079c 100644 --- a/tests/delta.py +++ b/tests/delta_test.py @@ -8,9 +8,6 @@ from threading import Thread -module_name = 'delta' - - class DeltaTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -21,17 +18,14 @@ def test_basic_delta_vacuum_truncate(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -79,9 +73,6 @@ def test_basic_delta_vacuum_truncate(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) - # @unittest.skip("skip") def test_delta_vacuum_truncate_1(self): """ @@ -90,18 +81,14 @@ def test_delta_vacuum_truncate_1(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -166,9 +153,6 @@ def test_delta_vacuum_truncate_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_vacuum_truncate_2(self): """ @@ -177,18 +161,14 @@ def test_delta_vacuum_truncate_2(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -206,7 +186,7 @@ def test_delta_vacuum_truncate_2(self): filepath = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')" - ).rstrip() + ).decode('utf-8').rstrip() self.backup_node(backup_dir, 'node', node) @@ -231,19 +211,15 @@ def test_delta_vacuum_truncate_2(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_stream(self): """ make archive node, take full and delta stream backups, restore them and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -263,7 +239,7 @@ def test_delta_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream']) @@ -274,7 +250,7 @@ def test_delta_stream(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream']) @@ -294,7 +270,7 @@ def test_delta_stream(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -310,13 +286,10 @@ def test_delta_stream(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_archive(self): """ @@ -324,10 +297,9 @@ def test_delta_archive(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -341,7 +313,7 @@ def test_delta_archive(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full') @@ -350,7 +322,7 @@ def test_delta_archive(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") - delta_result = node.execute("postgres", "SELECT * FROM t_heap") + delta_result = node.table_checksum("t_heap") delta_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='delta') @@ -369,7 +341,7 @@ def test_delta_archive(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -385,30 +357,25 @@ def test_delta_archive(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) node.slow_start() - delta_result_new = node.execute("postgres", "SELECT * FROM t_heap") + delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_multiple_segments(self): """ Make node, create table with multiple segments, write some data to it, check delta and data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', - 'autovacuum': 'off', 'full_page_writes': 'off' } ) @@ -433,25 +400,27 @@ def test_delta_multiple_segments(self): node.safe_psql("postgres", "checkpoint") # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") # delta BACKUP self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream']) + backup_dir, 'node', node, + backup_type='delta', options=['--stream']) # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') self.restore_node( - backup_dir, 'node', restored_node, options=[ - "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) + backup_dir, 'node', restored_node, + options=[ + "-j", "4", "-T", "{0}={1}".format( + tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM NODE_RESTORED pgdata_restored = self.pgdata_content(restored_node.data_dir) @@ -460,8 +429,7 @@ def test_delta_multiple_segments(self): self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') @@ -469,24 +437,22 @@ def test_delta_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_vacuum_full(self): """ make node, make full and delta stream backups, restore them and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.init_pb(backup_dir) @@ -514,7 +480,7 @@ def test_delta_vacuum_full(self): process.start() while not gdb.stopped_in_breakpoint: - sleep(1) + time.sleep(1) gdb.continue_execution_until_break(20) @@ -546,24 +512,19 @@ def test_delta_vacuum_full(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_create_db(self): """ Make node, take full backup, create database db1, take delta backup, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', - 'autovacuum': 'off' } ) @@ -577,7 +538,7 @@ def test_create_db(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -601,7 +562,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -672,25 +633,20 @@ def test_create_db(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exists_in_previous_backup(self): """ Make node, take full backup, create table, take page backup, take delta backup, check that file is no fully copied to delta backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', - 'autovacuum': 'off' } ) @@ -705,10 +661,10 @@ def test_exists_in_previous_backup(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") filepath = node.safe_psql( "postgres", - "SELECT pg_relation_filepath('t_heap')").rstrip() + "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', @@ -756,7 +712,7 @@ def test_exists_in_previous_backup(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -779,23 +735,18 @@ def test_exists_in_previous_backup(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_table_set_tablespace_delta(self): """ Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -821,8 +772,7 @@ def test_alter_table_set_tablespace_delta(self): "alter table t_heap set tablespace somedata_new") # DELTA BACKUP - result = node.safe_psql( - "postgres", "select * from t_heap") + result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, backup_type='delta', @@ -833,7 +783,7 @@ def test_alter_table_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -860,14 +810,10 @@ def test_alter_table_set_tablespace_delta(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from t_heap") + result_new = node_restored.table_checksum("t_heap") self.assertEqual(result, result_new, 'lost some data after restore') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_database_set_tablespace_delta(self): """ @@ -875,15 +821,11 @@ def test_alter_database_set_tablespace_delta(self): take delta backup, alter database tablespace location, take delta backup restore last delta backup. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -929,7 +871,7 @@ def test_alter_database_set_tablespace_delta(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -957,23 +899,18 @@ def test_alter_database_set_tablespace_delta(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_delta_delete(self): """ Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -1016,7 +953,7 @@ def test_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -1040,20 +977,16 @@ def test_delta_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_nullified_heap_page_backup(self): """ make node, take full backup, nullify some heap block, take delta backup, restore, physically compare pgdata`s """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1063,7 +996,7 @@ def test_delta_nullified_heap_page_backup(self): file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( "postgres", @@ -1104,7 +1037,7 @@ def test_delta_nullified_heap_page_backup(self): # Restore DELTA backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1114,21 +1047,17 @@ def test_delta_nullified_heap_page_backup(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_delta_backup_from_past(self): """ make node, take FULL stream backup, take DELTA stream backup, restore FULL backup, try to take second DELTA stream backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1169,23 +1098,18 @@ def test_delta_backup_from_past(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_delta_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1263,7 +1187,7 @@ def test_delta_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( @@ -1271,6 +1195,3 @@ def test_delta_pg_resetxlog(self): # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/exclude.py b/tests/exclude_test.py similarity index 58% rename from tests/exclude.py rename to tests/exclude_test.py index c9efe22af..cb3530cd5 100644 --- a/tests/exclude.py +++ b/tests/exclude_test.py @@ -3,10 +3,51 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'exclude' +class ExcludeTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + def test_exclude_temp_files(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'logging_collector': 'on', + 'log_filename': 'postgresql.log'}) -class ExcludeTest(ProbackupTest, unittest.TestCase): + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + oid = node.safe_psql( + 'postgres', + "select oid from pg_database where datname = 'postgres'").rstrip() + + pgsql_tmp_dir = os.path.join(node.data_dir, 'base', 'pgsql_tmp') + + os.mkdir(pgsql_tmp_dir) + + file = os.path.join(pgsql_tmp_dir, 'pgsql_tmp7351.16') + with open(file, 'w') as f: + f.write("HELLO") + f.flush() + f.close + + full_id = self.backup_node( + backup_dir, 'node', node, backup_type='full', options=['--stream']) + + file = os.path.join( + backup_dir, 'backups', 'node', full_id, + 'database', 'base', 'pgsql_tmp', 'pgsql_tmp7351.16') + + self.assertFalse( + os.path.exists(file), + "File must be excluded: {0}".format(file)) + + # TODO check temporary tablespaces # @unittest.skip("skip") # @unittest.expectedFailure @@ -15,10 +56,9 @@ def test_exclude_temp_tables(self): make node without archiving, create temp table, take full backup, check that temp table not present in backup catalogue """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -91,9 +131,6 @@ def test_exclude_temp_tables(self): "Found temp table file in backup catalogue.\n " "Filepath: {0}".format(file)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exclude_unlogged_tables_1(self): """ @@ -101,14 +138,12 @@ def test_exclude_unlogged_tables_1(self): alter table to unlogged, take delta backup, restore delta backup, check that PGDATA`s are physically the same """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', "shared_buffers": "10MB"}) self.init_pb(backup_dir) @@ -134,13 +169,12 @@ def test_exclude_unlogged_tables_1(self): self.backup_node( backup_dir, 'node', node, backup_type='delta', - options=['--stream'] - ) + options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -151,18 +185,92 @@ def test_exclude_unlogged_tables_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_exclude_unlogged_tables_2(self): + """ + 1. make node, create unlogged, take FULL, DELTA, PAGE, + check that unlogged table files was not backed up + 2. restore FULL, DELTA, PAGE to empty db, + ensure unlogged table exist and is epmty + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + "shared_buffers": "10MB"}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + backup_ids = [] + + for backup_type in ['full', 'delta', 'page']: + + if backup_type == 'full': + node.safe_psql( + 'postgres', + 'create unlogged table test as select generate_series(0,20050000)::text') + else: + node.safe_psql( + 'postgres', + 'insert into test select generate_series(0,20050000)::text') + + rel_path = node.execute( + 'postgres', + "select pg_relation_filepath('test')")[0][0] + + backup_id = self.backup_node( + backup_dir, 'node', node, + backup_type=backup_type, options=['--stream']) + + backup_ids.append(backup_id) + + filelist = self.get_backup_filelist( + backup_dir, 'node', backup_id) + + self.assertNotIn( + rel_path, filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.1', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.2', filelist, + "Unlogged table was not excluded") + + self.assertNotIn( + rel_path + '.3', filelist, + "Unlogged table was not excluded") + + # ensure restoring retrieves back only empty unlogged table + for backup_id in backup_ids: + node.stop() + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + + node.slow_start() + + self.assertEqual( + node.execute( + 'postgres', + 'select count(*) from test')[0][0], + 0) # @unittest.skip("skip") def test_exclude_log_dir(self): """ check that by default 'log' and 'pg_log' directories are not backed up """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -179,7 +287,7 @@ def test_exclude_log_dir(self): log_dir = node.safe_psql( 'postgres', - 'show log_directory').rstrip() + 'show log_directory').decode('utf-8').rstrip() node.cleanup() @@ -192,18 +300,14 @@ def test_exclude_log_dir(self): self.assertTrue(os.path.exists(path)) self.assertFalse(os.path.exists(log_file)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_exclude_log_dir_1(self): """ check that "--backup-pg-log" works correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -216,7 +320,7 @@ def test_exclude_log_dir_1(self): log_dir = node.safe_psql( 'postgres', - 'show log_directory').rstrip() + 'show log_directory').decode('utf-8').rstrip() self.backup_node( backup_dir, 'node', node, @@ -232,6 +336,3 @@ def test_exclude_log_dir_1(self): log_file = os.path.join(path, 'postgresql.log') self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(log_file)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 2170e2773..f0c77ae16 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -5,13 +5,14 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup version - pg_probackup init -B backup-path + pg_probackup init -B backup-dir - pg_probackup set-config -B backup-path --instance=instance_name + pg_probackup set-config -B backup-dir --instance=instance-name [-D pgdata-path] [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] @@ -31,29 +32,32 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--archive-port=port] [--archive-user=username] [--help] - pg_probackup set-backup -B backup-path --instance=instance_name + pg_probackup set-backup -B backup-dir --instance=instance-name -i backup-id [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup show-config -B backup-path --instance=instance_name + pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] + [--no-scale-units] [--help] - pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name [-D pgdata-path] [-C] - [--stream [-S slot-name]] [--temp-slot] + [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] [--no-validate] [--skip-block-validation] [--external-dirs=external-directories-paths] [--no-sync] [--log-level-console=log-level-console] [--log-level-file=log-level-file] + [--log-format-console=log-format-console] + [--log-format-file=log-format-file] [--log-filename=log-filename] [--error-log-filename=error-log-filename] [--log-directory=log-directory] [--log-rotation-size=log-rotation-size] - [--log-rotation-age=log-rotation-age] + [--log-rotation-age=log-rotation-age] [--no-color] [--delete-expired] [--delete-wal] [--merge-expired] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -70,7 +74,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ttl=interval] [--expire-time=timestamp] [--note=text] [--help] - pg_probackup restore -B backup-path --instance=instance_name + pg_probackup restore -B backup-dir --instance=instance-name [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -86,8 +90,10 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-T OLDDIR=NEWDIR] [--progress] [--external-mapping=OLDDIR=NEWDIR] [--skip-external-dirs] [--no-sync] + [-X WALDIR | --waldir=WALDIR] [-I | --incremental-mode=none|checksum|lsn] [--db-include | --db-exclude] + [--destroy-all-other-dbs] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] @@ -95,7 +101,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--archive-port=port] [--archive-user=username] [--help] - pg_probackup validate -B backup-path [--instance=instance_name] + pg_probackup validate -B backup-dir [--instance=instance-name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -104,45 +110,47 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--skip-block-validation] [--help] - pg_probackup checkdb [-B backup-path] [--instance=instance_name] + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] - [--heapallindexed] + [--heapallindexed] [--checkunique] [--help] - pg_probackup show -B backup-path - [--instance=instance_name [-i backup-id]] + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] [--format=format] [--archive] - [--help] + [--no-color] [--help] - pg_probackup delete -B backup-path --instance=instance_name + pg_probackup delete -B backup-dir --instance=instance-name [-j num-threads] [--progress] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] [--wal-depth=wal-depth] [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] [--delete-wal] - [--dry-run] + [--dry-run] [--no-validate] [--no-sync] [--help] - pg_probackup merge -B backup-path --instance=instance_name + pg_probackup merge -B backup-dir --instance=instance-name -i backup-id [--progress] [-j num-threads] + [--no-validate] [--no-sync] [--help] - pg_probackup add-instance -B backup-path -D pgdata-path - --instance=instance_name + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--help] - pg_probackup del-instance -B backup-path - --instance=instance_name + pg_probackup del-instance -B backup-dir + --instance=instance-name [--help] - pg_probackup archive-push -B backup-path --instance=instance_name + pg_probackup archive-push -B backup-dir --instance=instance-name --wal-file-name=wal-file-name + [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] [--archive-timeout=timeout] [--no-ready-rename] [--no-sync] @@ -154,7 +162,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--help] - pg_probackup archive-get -B backup-path --instance=instance_name + pg_probackup archive-get -B backup-dir --instance=instance-name --wal-file-path=wal-file-path --wal-file-name=wal-file-name [-j num-threads] [--batch-size=batch_size] @@ -164,5 +172,20 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--help] -Read the website for details. + pg_probackup catchup -b catchup-mode + --source-pgdata=path_to_pgdata_on_remote_server + --destination-pgdata=path_to_local_dir + [--stream [-S slot-name] [--temp-slot | --perm-slot]] + [-j num-threads] + [-T OLDDIR=NEWDIR] + [--exclude-path=path_prefix] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--dry-run] + [--help] + +Read the website for details . Report bugs to . diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out new file mode 100644 index 000000000..bd6d76970 --- /dev/null +++ b/tests/expected/option_help_ru.out @@ -0,0 +1,191 @@ + +pg_probackup - утилита для управления резервным копированием/восстановлением базы данных PostgreSQL. + + pg_probackup help [COMMAND] + + pg_probackup version + + pg_probackup init -B backup-dir + + pg_probackup set-config -B backup-dir --instance=instance-name + [-D pgdata-path] + [--external-dirs=external-directories-paths] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-format-file=log-format-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--archive-timeout=timeout] + [-d dbname] [-h host] [-p port] [-U username] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--restore-command=cmdline] [--archive-host=destination] + [--archive-port=port] [--archive-user=username] + [--help] + + pg_probackup set-backup -B backup-dir --instance=instance-name + -i backup-id [--ttl=interval] [--expire-time=timestamp] + [--note=text] + [--help] + + pg_probackup show-config -B backup-dir --instance=instance-name + [--format=format] + [--no-scale-units] + [--help] + + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name + [-D pgdata-path] [-C] + [--stream [-S slot-name] [--temp-slot]] + [--backup-pg-log] [-j num-threads] [--progress] + [--no-validate] [--skip-block-validation] + [--external-dirs=external-directories-paths] + [--no-sync] + [--log-level-console=log-level-console] + [--log-level-file=log-level-file] + [--log-format-console=log-format-console] + [--log-format-file=log-format-file] + [--log-filename=log-filename] + [--error-log-filename=error-log-filename] + [--log-directory=log-directory] + [--log-rotation-size=log-rotation-size] + [--log-rotation-age=log-rotation-age] [--no-color] + [--delete-expired] [--delete-wal] [--merge-expired] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--archive-timeout=archive-timeout] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--help] + + pg_probackup restore -B backup-dir --instance=instance-name + [-D pgdata-path] [-i backup-id] [-j num-threads] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] + [--recovery-target=immediate|latest] + [--recovery-target-name=target-name] + [--recovery-target-action=pause|promote|shutdown] + [--restore-command=cmdline] + [-R | --restore-as-replica] [--force] + [--primary-conninfo=primary_conninfo] + [-S | --primary-slot-name=slotname] + [--no-validate] [--skip-block-validation] + [-T OLDDIR=NEWDIR] [--progress] + [--external-mapping=OLDDIR=NEWDIR] + [--skip-external-dirs] [--no-sync] + [-X WALDIR | --waldir=WALDIR] + [-I | --incremental-mode=none|checksum|lsn] + [--db-include | --db-exclude] + [--destroy-all-other-dbs] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--archive-host=hostname] + [--archive-port=port] [--archive-user=username] + [--help] + + pg_probackup validate -B backup-dir [--instance=instance-name] + [-i backup-id] [--progress] [-j num-threads] + [--recovery-target-time=time|--recovery-target-xid=xid + |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] + [--recovery-target-timeline=timeline] + [--recovery-target-name=target-name] + [--skip-block-validation] + [--help] + + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] + [-D pgdata-path] [--progress] [-j num-threads] + [--amcheck] [--skip-block-validation] + [--heapallindexed] [--checkunique] + [--help] + + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] + [--format=format] [--archive] + [--no-color] [--help] + + pg_probackup delete -B backup-dir --instance=instance-name + [-j num-threads] [--progress] + [--retention-redundancy=retention-redundancy] + [--retention-window=retention-window] + [--wal-depth=wal-depth] + [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] + [--delete-wal] + [--dry-run] [--no-validate] [--no-sync] + [--help] + + pg_probackup merge -B backup-dir --instance=instance-name + -i backup-id [--progress] [-j num-threads] + [--no-validate] [--no-sync] + [--help] + + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name + [--external-dirs=external-directories-paths] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup del-instance -B backup-dir + --instance=instance-name + [--help] + + pg_probackup archive-push -B backup-dir --instance=instance-name + --wal-file-name=wal-file-name + [--wal-file-path=wal-file-path] + [-j num-threads] [--batch-size=batch_size] + [--archive-timeout=timeout] + [--no-ready-rename] [--no-sync] + [--overwrite] [--compress] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup archive-get -B backup-dir --instance=instance-name + --wal-file-path=wal-file-path + --wal-file-name=wal-file-name + [-j num-threads] [--batch-size=batch_size] + [--no-validate-wal] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--help] + + pg_probackup catchup -b catchup-mode + --source-pgdata=path_to_pgdata_on_remote_server + --destination-pgdata=path_to_local_dir + [--stream [-S slot-name] [--temp-slot | --perm-slot]] + [-j num-threads] + [-T OLDDIR=NEWDIR] + [--exclude-path=path_prefix] + [-d dbname] [-h host] [-p port] [-U username] + [-w --no-password] [-W --password] + [--remote-proto] [--remote-host] + [--remote-port] [--remote-path] [--remote-user] + [--ssh-options] + [--dry-run] + [--help] + +Подробнее читайте на сайте . +Сообщайте об ошибках в . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out deleted file mode 100644 index 3a8236d68..000000000 --- a/tests/expected/option_version.out +++ /dev/null @@ -1 +0,0 @@ -pg_probackup 2.4.1 \ No newline at end of file diff --git a/tests/external.py b/tests/external_test.py similarity index 86% rename from tests/external.py rename to tests/external_test.py index 9d14d7558..53f3c5449 100644 --- a/tests/external.py +++ b/tests/external_test.py @@ -6,8 +6,6 @@ import shutil -module_name = 'external' - # TODO: add some ptrack tests class ExternalTest(ProbackupTest, unittest.TestCase): @@ -19,15 +17,14 @@ def test_basic_external(self): with external directory, restore backup, check that external directory was successfully copied """ - fname = self.id().split('.')[3] - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory @@ -38,7 +35,8 @@ def test_basic_external(self): # take FULL backup with external directory pointing to a file file_path = os.path.join(core_dir, 'file') - open(file_path, "w+") + with open(file_path, "w+") as f: + pass try: self.backup_node( @@ -90,9 +88,6 @@ def test_basic_external(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_external_none(self): @@ -102,13 +97,12 @@ def test_external_none(self): restore delta backup, check that external directory was not copied """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory @@ -152,9 +146,6 @@ def test_external_none(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_external_dirs_overlapping(self): @@ -163,13 +154,12 @@ def test_external_dirs_overlapping(self): take backup with two external directories pointing to the same directory, backup should fail """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') @@ -206,9 +196,6 @@ def test_external_dirs_overlapping(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_external_dir_mapping(self): """ @@ -217,13 +204,12 @@ def test_external_dir_mapping(self): check that restore with external-dir mapping will end with success """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -246,7 +232,7 @@ def test_external_dir_mapping(self): data_dir=external_dir2, options=["-j", "4"]) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') @@ -299,20 +285,16 @@ def test_external_dir_mapping(self): node_restored.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_backup_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -360,9 +342,6 @@ def test_backup_multiple_external(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility(self): @@ -372,13 +351,14 @@ def test_external_backward_compatibility(self): restore delta backup, check that incremental chain restored correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -444,7 +424,7 @@ def test_external_backward_compatibility(self): # RESTORE chain with new binary node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -463,9 +443,6 @@ def test_external_backward_compatibility(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility_merge_1(self): @@ -474,13 +451,14 @@ def test_external_backward_compatibility_merge_1(self): take delta backup with new binary and 2 external directories merge delta backup ajd restore it """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -537,7 +515,7 @@ def test_external_backward_compatibility_merge_1(self): # Restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -556,9 +534,6 @@ def test_external_backward_compatibility_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_backward_compatibility_merge_2(self): @@ -567,13 +542,14 @@ def test_external_backward_compatibility_merge_2(self): take delta backup with new binary and 2 external directories merge delta backup and restore it """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir, old_binary=True) self.show_pb(backup_dir) @@ -659,7 +635,7 @@ def test_external_backward_compatibility_merge_2(self): # Restore merged backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -682,20 +658,18 @@ def test_external_backward_compatibility_merge_2(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node, old_binary=True) @@ -770,21 +744,15 @@ def test_external_merge(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_skip_external_dirs(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -870,21 +838,15 @@ def test_external_merge_skip_external_dirs(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_1(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -952,20 +914,15 @@ def test_external_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_3(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1046,21 +1003,15 @@ def test_external_merge_3(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_merge_2(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1142,21 +1093,15 @@ def test_external_merge_2(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_external_changed_data(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1242,21 +1187,16 @@ def test_restore_external_changed_data(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_external_changed_data_1(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'max_wal_size': '32MB'}) self.init_pb(backup_dir) @@ -1350,21 +1290,16 @@ def test_restore_external_changed_data_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_merge_external_changed_data(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'max_wal_size': '32MB'}) self.init_pb(backup_dir) @@ -1454,23 +1389,17 @@ def test_merge_external_changed_data(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_restore_skip_external(self): """ Check that --skip-external-dirs works correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1527,9 +1456,6 @@ def test_restore_skip_external(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_is_symlink(self): @@ -1539,18 +1465,15 @@ def test_external_dir_is_symlink(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1563,7 +1486,7 @@ def test_external_dir_is_symlink(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1587,7 +1510,7 @@ def test_external_dir_is_symlink(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1612,9 +1535,6 @@ def test_external_dir_is_symlink(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_contain_symlink_on_dir(self): @@ -1624,14 +1544,13 @@ def test_external_dir_contain_symlink_on_dir(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1647,7 +1566,7 @@ def test_external_dir_contain_symlink_on_dir(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1672,7 +1591,7 @@ def test_external_dir_contain_symlink_on_dir(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1697,9 +1616,6 @@ def test_external_dir_contain_symlink_on_dir(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_contain_symlink_on_file(self): @@ -1709,14 +1625,13 @@ def test_external_dir_contain_symlink_on_file(self): but restored as directory """ if os.name == 'nt': - return unittest.skip('Skipped for Windows') + self.skipTest('Skipped for Windows') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1732,7 +1647,7 @@ def test_external_dir_contain_symlink_on_file(self): backup_dir, 'node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, module_name, fname) + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) symlinked_dir = os.path.join(core_dir, 'symlinked') self.restore_node( @@ -1745,7 +1660,7 @@ def test_external_dir_contain_symlink_on_file(self): # create symlink to directory in external directory src_file = os.path.join(symlinked_dir, 'postgresql.conf') os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) os.symlink(src_file, file_in_external_dir) # FULL backup with external directories @@ -1759,7 +1674,7 @@ def test_external_dir_contain_symlink_on_file(self): node.base_dir, exclude_dirs=['logs']) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # RESTORE node_restored.cleanup() @@ -1784,9 +1699,6 @@ def test_external_dir_contain_symlink_on_file(self): backup_dir, 'node', backup_id=backup_id)['external-dirs']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_external_dir_is_tablespace(self): @@ -1794,16 +1706,13 @@ def test_external_dir_is_tablespace(self): Check that backup fails with error if external directory points to tablespace """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1836,25 +1745,19 @@ def test_external_dir_is_tablespace(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_not_empty(self): """ Check that backup fails with error if external directory point to not empty tablespace and if remapped directory also isn`t empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1918,9 +1821,6 @@ def test_restore_external_dir_not_empty(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_is_missing(self): """ take FULL backup with not empty external directory @@ -1928,16 +1828,13 @@ def test_restore_external_dir_is_missing(self): take DELTA backup with external directory, which should fail """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2002,9 +1899,6 @@ def test_restore_external_dir_is_missing(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_is_missing(self): """ take FULL backup with not empty external directory @@ -2015,16 +1909,13 @@ def test_merge_external_dir_is_missing(self): merge it into FULL, restore and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2092,9 +1983,6 @@ def test_merge_external_dir_is_missing(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_is_empty(self): """ take FULL backup with not empty external directory @@ -2103,16 +1991,13 @@ def test_restore_external_dir_is_empty(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2123,7 +2008,7 @@ def test_restore_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2158,9 +2043,6 @@ def test_restore_external_dir_is_empty(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_is_empty(self): """ take FULL backup with not empty external directory @@ -2169,16 +2051,13 @@ def test_merge_external_dir_is_empty(self): merge backups and restore FULL, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2189,7 +2068,7 @@ def test_merge_external_dir_is_empty(self): # create empty file in external directory # open(os.path.join(external_dir, 'file'), 'a').close() os.mkdir(external_dir) - os.chmod(external_dir, 0700) + os.chmod(external_dir, 0o0700) with open(os.path.join(external_dir, 'file'), 'w+') as f: f.close() @@ -2227,9 +2106,6 @@ def test_merge_external_dir_is_empty(self): node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_restore_external_dir_string_order(self): """ take FULL backup with not empty external directory @@ -2238,16 +2114,13 @@ def test_restore_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2258,12 +2131,12 @@ def test_restore_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0700) + os.chmod(external_dir_1, 0o0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0700) + os.chmod(external_dir_2, 0o0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() @@ -2309,9 +2182,6 @@ def test_restore_external_dir_string_order(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_external_dir_string_order(self): """ take FULL backup with not empty external directory @@ -2320,16 +2190,13 @@ def test_merge_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - core_dir = os.path.join(self.tmp_path, module_name, fname) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) shutil.rmtree(core_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2340,12 +2207,12 @@ def test_merge_external_dir_string_order(self): # create empty file in external directory os.mkdir(external_dir_1) - os.chmod(external_dir_1, 0700) + os.chmod(external_dir_1, 0o0700) with open(os.path.join(external_dir_1, 'fileA'), 'w+') as f: f.close() os.mkdir(external_dir_2) - os.chmod(external_dir_2, 0700) + os.chmod(external_dir_2, 0o0700) with open(os.path.join(external_dir_2, 'fileZ'), 'w+') as f: f.close() @@ -2394,9 +2261,6 @@ def test_merge_external_dir_string_order(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smart_restore_externals(self): """ @@ -2405,13 +2269,12 @@ def test_smart_restore_externals(self): make sure that files from externals are not copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2458,7 +2321,7 @@ def test_smart_restore_externals(self): logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = f.read() # get delta between FULL and PAGE filelists filelist_full = self.get_backup_filelist( @@ -2473,9 +2336,6 @@ def test_smart_restore_externals(self): for file in filelist_diff: self.assertNotIn(file, logfile_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_external_validation(self): """ @@ -2484,13 +2344,12 @@ def test_external_validation(self): corrupt external file in backup, run validate which should fail """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2544,6 +2403,3 @@ def test_external_validation(self): 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], 'Backup STATUS should be "CORRUPT"') - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/false_positive.py b/tests/false_positive_test.py similarity index 55% rename from tests/false_positive.py rename to tests/false_positive_test.py index fc9ee4b62..ea82cb18f 100644 --- a/tests/false_positive.py +++ b/tests/false_positive_test.py @@ -1,13 +1,12 @@ import unittest import os +from time import sleep + from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta import subprocess -module_name = 'false_positive' - - class FalsePositive(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -16,13 +15,12 @@ def test_validate_wal_lost_segment(self): """ Loose segment located between backups. ExpectedFailure. This is BUG """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -47,19 +45,15 @@ def test_validate_wal_lost_segment(self): backup_dir, 'node')) ######## - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.expectedFailure # Need to force validation of ancestor-chain def test_incremental_backup_corrupt_full_1(self): """page-level backup with corrupted full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -83,12 +77,10 @@ def test_incremental_backup_corrupt_full_1(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: Valid backup on current timeline is not found. ' + 'ERROR: Valid full backup on current timeline is not found. ' 'Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - sleep(1) self.assertFalse( True, "Expecting Error because page backup should not be " @@ -98,7 +90,7 @@ def test_incremental_backup_corrupt_full_1(self): except ProbackupException as e: self.assertEqual( e.message, - 'ERROR: Valid backup on current timeline is not found. ' + 'ERROR: Valid full backup on current timeline is not found. ' 'Create new FULL backup before an incremental one.\n', '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -106,195 +98,6 @@ def test_incremental_backup_corrupt_full_1(self): self.assertEqual( self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.expectedFailure - def test_ptrack_concurrent_get_and_clear_1(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'], - gdb=True - ) - - gdb.set_breakpoint('make_pagemap_from_ptrack') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - tablespace_oid = node.safe_psql( - "postgres", - "select oid from pg_tablespace where spcname = 'pg_default'").rstrip() - - relfilenode = node.safe_psql( - "postgres", - "select 't_heap'::regclass::oid").rstrip() - - node.safe_psql( - "postgres", - "SELECT pg_ptrack_get_and_clear({0}, {1})".format( - tablespace_oid, relfilenode)) - - gdb.continue_execution_until_exit() - - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - # Logical comparison - self.assertEqual( - result, - node.safe_psql("postgres", "SELECT * FROM t_heap")) - - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.expectedFailure - def test_ptrack_concurrent_get_and_clear_2(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - - if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') - - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - node.safe_psql( - "postgres", - "create table t_heap as select i" - " as id from generate_series(0,1) i" - ) - - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=['--stream'], - gdb=True - ) - - gdb.set_breakpoint('pthread_create') - gdb.run_until_break() - - node.safe_psql( - "postgres", - "update t_heap set id = 100500") - - tablespace_oid = node.safe_psql( - "postgres", - "select oid from pg_tablespace " - "where spcname = 'pg_default'").rstrip() - - relfilenode = node.safe_psql( - "postgres", - "select 't_heap'::regclass::oid").rstrip() - - node.safe_psql( - "postgres", - "SELECT pg_ptrack_get_and_clear({0}, {1})".format( - tablespace_oid, relfilenode)) - - gdb._execute("delete breakpoints") - gdb.continue_execution_until_exit() - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream'] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup ptrack_lsn.\n" - " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - if self.paranoia: - pgdata = self.pgdata_content(node.data_dir) - - result = node.safe_psql("postgres", "SELECT * FROM t_heap") - node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) - - # Physical comparison - if self.paranoia: - pgdata_restored = self.pgdata_content( - node.data_dir, ignore_ptrack=False) - self.compare_pgdata(pgdata, pgdata_restored) - - node.slow_start() - # Logical comparison - self.assertEqual( - result, - node.safe_psql("postgres", "SELECT * FROM t_heap") - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_pg_10_waldir(self): @@ -302,20 +105,20 @@ def test_pg_10_waldir(self): test group access for PG >= 11 """ if self.pg_config_version < self.version_to_num('10.0'): - return unittest.skip('You need PostgreSQL >= 10 for this test') + self.skipTest('You need PostgreSQL >= 10 for this test') - fname = self.id().split('.')[3] wal_dir = os.path.join( - os.path.join(self.tmp_path, module_name, fname), 'wal_dir') + os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') + import shutil shutil.rmtree(wal_dir, ignore_errors=True) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', '--waldir={0}'.format(wal_dir)]) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -328,7 +131,7 @@ def test_pg_10_waldir(self): # restore backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -342,9 +145,6 @@ def test_pg_10_waldir(self): os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), 'pg_wal should be symlink') - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_time_backup_victim(self): @@ -353,10 +153,9 @@ def test_recovery_target_time_backup_victim(self): probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -386,6 +185,7 @@ def test_recovery_target_time_backup_victim(self): gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -403,21 +203,20 @@ def test_recovery_target_time_backup_victim(self): backup_dir, 'node', options=['--recovery-target-time={0}'.format(target_time)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") def test_recovery_target_lsn_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 + + @y.sokolov: looks like this test should pass. + So I commented 'expectedFailure' """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -445,6 +244,7 @@ def test_recovery_target_lsn_backup_victim(self): backup_dir, 'node', node, options=['--log-level-console=LOG'], gdb=True) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -466,9 +266,6 @@ def test_recovery_target_lsn_backup_victim(self): backup_dir, 'node', options=['--recovery-target-lsn={0}'.format(target_lsn)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_streaming_timeout(self): @@ -477,10 +274,9 @@ def test_streaming_timeout(self): message because our WAL streaming engine is "borrowed" from pg_receivexlog """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -496,6 +292,7 @@ def test_streaming_timeout(self): backup_dir, 'node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) + # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() @@ -516,20 +313,16 @@ def test_streaming_timeout(self): 'ERROR: Problem in receivexlog', log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") @unittest.expectedFailure def test_validate_all_empty_catalog(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) try: @@ -545,6 +338,3 @@ def test_validate_all_empty_catalog(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index ac64c4230..2e5ed40e8 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,2 +1,9 @@ -__all__ = ['ptrack_helpers', 'cfs_helpers', 'expected_errors'] -#from . import * \ No newline at end of file +__all__ = ['ptrack_helpers', 'cfs_helpers', 'data_helpers'] + +import unittest + +# python 2.7 compatibility +if not hasattr(unittest.TestCase, "skipTest"): + def skipTest(self, reason): + raise unittest.SkipTest(reason) + unittest.TestCase.skipTest = skipTest \ No newline at end of file diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py index 67e2b331b..31af76f2e 100644 --- a/tests/helpers/cfs_helpers.py +++ b/tests/helpers/cfs_helpers.py @@ -88,4 +88,6 @@ def corrupt_file(filename): def random_string(n): a = string.ascii_letters + string.digits - return ''.join([random.choice(a) for i in range(int(n)+1)]) \ No newline at end of file + random_str = ''.join([random.choice(a) for i in range(int(n)+1)]) + return str.encode(random_str) +# return ''.join([random.choice(a) for i in range(int(n)+1)]) diff --git a/tests/helpers/data_helpers.py b/tests/helpers/data_helpers.py new file mode 100644 index 000000000..27cb66c3d --- /dev/null +++ b/tests/helpers/data_helpers.py @@ -0,0 +1,78 @@ +import re +import unittest +import functools +import time + +def _tail_file(file, linetimeout, totaltimeout): + start = time.time() + with open(file, 'r') as f: + waits = 0 + while waits < linetimeout: + line = f.readline() + if line == '': + waits += 1 + time.sleep(1) + continue + waits = 0 + yield line + if time.time() - start > totaltimeout: + raise TimeoutError("total timeout tailing %s" % (file,)) + else: + raise TimeoutError("line timeout tailing %s" % (file,)) + + +class tail_file(object): # snake case to immitate function + def __init__(self, filename, *, linetimeout=10, totaltimeout=60, collect=False): + self.filename = filename + self.tailer = _tail_file(filename, linetimeout, totaltimeout) + self.collect = collect + self.lines = [] + self._content = None + + def __iter__(self): + return self + + def __next__(self): + line = next(self.tailer) + if self.collect: + self.lines.append(line) + self._content = None + return line + + @property + def content(self): + if not self.collect: + raise AttributeError("content collection is not enabled", + name="content", obj=self) + if not self._content: + self._content = "".join(self.lines) + return self._content + + def drop_content(self): + self.lines.clear() + self._content = None + + def stop_collect(self): + self.drop_content() + self.collect = False + + def wait(self, *, contains:str = None, regex:str = None): + assert contains != None or regex != None + assert contains == None or regex == None + try: + for line in self: + if contains is not None and contains in line: + break + if regex is not None and re.search(regex, line): + break + except TimeoutError: + msg = "Didn't found expected " + if contains is not None: + msg += repr(contains) + elif regex is not None: + msg += f"/{regex}/" + msg += f" in {self.filename}" + raise unittest.TestCase.failureException(msg) + + def wait_shutdown(self): + self.wait(contains='database system is shut down') diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 19d399d4b..27d982856 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,6 +1,9 @@ # you need os for unittest to work import os +import gc +import unittest from sys import exit, argv, version_info +import signal import subprocess import shutil import six @@ -12,6 +15,7 @@ from time import sleep import re import json +import random idx_ptrack = { 't_heap': { @@ -86,25 +90,56 @@ def dir_files(base_dir): return out_list +def is_pgpro(): + # pg_config --help + cmd = [os.environ['PG_CONFIG'], '--help'] + + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return b'postgrespro' in result.stdout + + def is_enterprise(): # pg_config --help - if os.name == 'posix': - cmd = [os.environ['PG_CONFIG'], '--help'] - - elif os.name == 'nt': - cmd = [[os.environ['PG_CONFIG']], ['--help']] - - p = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - if b'postgrespro.ru' in p.communicate()[0]: - return True - else: + cmd = [os.environ['PG_CONFIG'], '--help'] + + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + # PostgresPro std or ent + if b'postgrespro' in p.stdout: + cmd = [os.environ['PG_CONFIG'], '--pgpro-edition'] + p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + + return b'enterprise' in p.stdout + else: # PostgreSQL return False +def is_nls_enabled(): + cmd = [os.environ['PG_CONFIG'], '--configure'] + + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return b'enable-nls' in result.stdout + + +def base36enc(number): + """Converts an integer to a base36 string.""" + alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + base36 = '' + sign = '' + + if number < 0: + sign = '-' + number = -number + + if 0 <= number < len(alphabet): + return sign + alphabet[number] + + while number != 0: + number, i = divmod(number, len(alphabet)) + base36 = alphabet[i] + base36 + + return sign + base36 + + class ProbackupException(Exception): def __init__(self, message, cmd): self.message = message @@ -113,41 +148,105 @@ def __init__(self, message, cmd): def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +class PostgresNodeExtended(testgres.PostgresNode): -def slow_start(self, replica=False): + def __init__(self, base_dir=None, *args, **kwargs): + super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs) + self.is_started = False - # wait for https://github.com/postgrespro/testgres/pull/50 -# self.start() -# self.poll_query_until( -# "postgres", -# "SELECT not pg_is_in_recovery()", -# suppress={testgres.NodeConnection}) - if replica: - query = 'SELECT pg_is_in_recovery()' - else: - query = 'SELECT not pg_is_in_recovery()' + def slow_start(self, replica=False): - self.start() - while True: - try: - output = self.safe_psql('template1', query).rstrip() + # wait for https://github.com/postgrespro/testgres/pull/50 + # self.start() + # self.poll_query_until( + # "postgres", + # "SELECT not pg_is_in_recovery()", + # suppress={testgres.NodeConnection}) + if replica: + query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' - if output == 't': + self.start() + while True: + try: + output = self.safe_psql('template1', query).decode("utf-8").rstrip() + + if output == 't': + break + + except testgres.QueryException as e: + if 'database system is starting up' in e.message: + pass + elif 'FATAL: the database system is not accepting connections' in e.message: + pass + elif replica and 'Hot standby mode is disabled' in e.message: + raise e + else: + raise e + + sleep(0.5) + + def start(self, *args, **kwargs): + if not self.is_started: + super(PostgresNodeExtended, self).start(*args, **kwargs) + self.is_started = True + return self + + def stop(self, *args, **kwargs): + if self.is_started: + result = super(PostgresNodeExtended, self).stop(*args, **kwargs) + self.is_started = False + return result + + def kill(self, someone = None): + if self.is_started: + sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK + if someone == None: + os.kill(self.pid, sig) + else: + os.kill(self.auxiliary_pids[someone][0], sig) + self.is_started = False + + def table_checksum(self, table, dbname="postgres"): + con = self.connect(dbname=dbname) + + curname = "cur_"+str(random.randint(0,2**48)) + + con.execute(""" + DECLARE %s NO SCROLL CURSOR FOR + SELECT t::text FROM %s as t + """ % (curname, table)) + + sum = hashlib.md5() + while True: + rows = con.execute("FETCH FORWARD 5000 FROM %s" % curname) + if not rows: break + for row in rows: + # hash uses SipHash since Python3.4, therefore it is good enough + sum.update(row[0].encode('utf8')) - except testgres.QueryException as e: - if 'database system is starting up' in e[0]: - continue - else: - raise e + con.execute("CLOSE %s; ROLLBACK;" % curname) + con.close() + return sum.hexdigest() class ProbackupTest(object): # Class attributes enterprise = is_enterprise() + enable_nls = is_nls_enabled() + pgpro = is_pgpro() def __init__(self, *args, **kwargs): super(ProbackupTest, self).__init__(*args, **kwargs) + + self.nodes_to_cleanup = [] + + if isinstance(self, unittest.TestCase): + self.module_name = self.id().split('.')[1] + self.fname = self.id().split('.')[3] + if '-v' in argv or '--verbose' in argv: self.verbose = True else: @@ -178,15 +277,15 @@ def __init__(self, *args, **kwargs): self.test_env['LC_MESSAGES'] = 'C' self.test_env['LC_TIME'] = 'C' - self.paranoia = False - if 'PG_PROBACKUP_PARANOIA' in self.test_env: - if self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON': - self.paranoia = True + self.gdb = 'PGPROBACKUP_GDB' in self.test_env and \ + self.test_env['PGPROBACKUP_GDB'] == 'ON' + + self.paranoia = 'PG_PROBACKUP_PARANOIA' in self.test_env and \ + self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON' + + self.archive_compress = 'ARCHIVE_COMPRESSION' in self.test_env and \ + self.test_env['ARCHIVE_COMPRESSION'] == 'ON' - self.archive_compress = False - if 'ARCHIVE_COMPRESSION' in self.test_env: - if self.test_env['ARCHIVE_COMPRESSION'] == 'ON': - self.archive_compress = True try: testgres.configure_testgres( cache_initdb=False, @@ -211,10 +310,7 @@ def __init__(self, *args, **kwargs): self.user = self.get_username() self.probackup_path = None if 'PGPROBACKUPBIN' in self.test_env: - if ( - os.path.isfile(self.test_env["PGPROBACKUPBIN"]) and - os.access(self.test_env["PGPROBACKUPBIN"], os.X_OK) - ): + if shutil.which(self.test_env["PGPROBACKUPBIN"]): self.probackup_path = self.test_env["PGPROBACKUPBIN"] else: if self.verbose: @@ -246,18 +342,6 @@ def __init__(self, *args, **kwargs): print('pg_probackup binary is not found') exit(1) - self.probackup_version = None - - try: - self.probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8')) - - self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) - if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' os.environ['PATH'] = os.path.dirname( @@ -280,6 +364,32 @@ def __init__(self, *args, **kwargs): if self.verbose: print('PGPROBACKUPBIN_OLD is not an executable file') + self.probackup_version = None + self.old_probackup_version = None + + try: + self.probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8')) + + if self.probackup_old_path: + old_probackup_version_output = subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.old_probackup_version = re.search( + r"\d+\.\d+\.\d+", + subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + ).group(0) + + self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) + self.remote = False self.remote_host = None self.remote_port = None @@ -292,10 +402,54 @@ def __init__(self, *args, **kwargs): self.ptrack = False if 'PG_PROBACKUP_PTRACK' in self.test_env: if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': - self.ptrack = True + if self.pg_config_version >= self.version_to_num('11.0'): + self.ptrack = True os.environ["PGAPPNAME"] = "pg_probackup" + def is_test_result_ok(test_case): + # sources of solution: + # 1. python versions 2.7 - 3.10, verified on 3.10, 3.7, 2.7, taken from: + # https://tousu.in/qa/?qa=555402/unit-testing-getting-pythons-unittest-results-in-a-teardown-method&show=555403#a555403 + # + # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 + + if not isinstance(test_case, unittest.TestCase): + raise AssertionError("test_case is not instance of unittest.TestCase") + + if hasattr(test_case, '_outcome'): # Python 3.4+ + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ and pytest 5.3.5+ + result = test_case._outcome.result + if not hasattr(result, 'errors'): + result.errors = [] + if not hasattr(result, 'failures'): + result.failures = [] + else: # Python 2.7, 3.0-3.3 + result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + + ok = all(test != test_case for test, text in result.errors + result.failures) + + return ok + + def tearDown(self): + if self.is_test_result_ok(): + for node in self.nodes_to_cleanup: + node.cleanup() + self.del_test_dir(self.module_name, self.fname) + + else: + for node in self.nodes_to_cleanup: + # TODO make decorator with proper stop() vs cleanup() + node._try_shutdown(max_attempts=1) + # node.cleanup() + + self.nodes_to_cleanup.clear() + @property def pg_config_version(self): return self.version_to_num( @@ -319,6 +473,19 @@ def pg_config_version(self): # print('PGPROBACKUP_SSH_USER is not set') # exit(1) + def make_empty_node( + self, + base_dir=None): + real_base_dir = os.path.join(self.tmp_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = PostgresNodeExtended(base_dir=real_base_dir) + node.should_rm_dirs = True + self.nodes_to_cleanup.append(node) + + return node + def make_simple_node( self, base_dir=None, @@ -327,14 +494,7 @@ def make_simple_node( initdb_params=[], pg_options={}): - real_base_dir = os.path.join(self.tmp_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - os.makedirs(real_base_dir) - - node = testgres.get_new_node('test', base_dir=real_base_dir) - # bound method slow_start() to 'node' class instance - node.slow_start = slow_start.__get__(node) - node.should_rm_dirs = True + node = self.make_empty_node(base_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) @@ -359,17 +519,20 @@ def make_simple_node( options['log_connections'] = 'on' options['log_disconnections'] = 'on' options['restart_after_crash'] = 'off' + options['autovacuum'] = 'off' # Allow replication in pg_hba.conf if set_replication: options['max_wal_senders'] = 10 if ptrack_enable: - if node.major_version > 11: - options['ptrack.map_size'] = '128' - options['shared_preload_libraries'] = 'ptrack' - else: - options['ptrack_enable'] = 'on' + options['ptrack.map_size'] = '128' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '100' # set default values self.set_auto_conf(node, options) @@ -377,7 +540,83 @@ def make_simple_node( # Apply given parameters self.set_auto_conf(node, pg_options) + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + self.set_auto_conf( + node, {}, 'postgresql.conf', ['wal_keep_segments']) + return node + + def simple_bootstrap(self, node, role) -> None: + + node.safe_psql( + 'postgres', + 'CREATE ROLE {0} WITH LOGIN REPLICATION'.format(role)) + + # PG 9.5 + if self.get_version(node) < 90600: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0};'.format(role)) + # PG 9.6 + elif self.get_version(node) > 90600 and self.get_version(node) < 100000: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) + # >= 15 + else: + node.safe_psql( + 'postgres', + 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' + 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False): res = node.execute( @@ -408,6 +647,35 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False # res[0], 0, # 'Failed to create tablespace with cmd: {0}'.format(cmd)) + def drop_tblspace(self, node, tblspc_name): + res = node.execute( + 'postgres', + 'select exists' + " (select 1 from pg_tablespace where spcname = '{0}')".format( + tblspc_name) + ) + # Check that tablespace with name 'tblspc_name' do not exists already + self.assertTrue( + res[0][0], + 'Tablespace "{0}" do not exists'.format(tblspc_name) + ) + + rels = node.execute( + "postgres", + "SELECT relname FROM pg_class c " + "LEFT JOIN pg_tablespace t ON c.reltablespace = t.oid " + "where c.relkind = 'r' and t.spcname = '{0}'".format(tblspc_name)) + + for rel in rels: + node.safe_psql( + 'postgres', + "DROP TABLE {0}".format(rel[0])) + + node.safe_psql( + 'postgres', + 'DROP TABLESPACE {0}'.format(tblspc_name)) + + def get_tblspace_path(self, node, tblspc_name): return os.path.join(node.base_dir, tblspc_name) @@ -427,7 +695,8 @@ def get_fork_path(self, node, fork_name): def get_md5_per_page_for_fork(self, file, size_in_pages): pages_per_segment = {} md5_per_page = {} - nsegments = size_in_pages/131072 + size_in_pages = int(size_in_pages) + nsegments = int(size_in_pages/131072) if size_in_pages % 131072 != 0: nsegments = nsegments + 1 @@ -479,10 +748,10 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): if os.path.getsize(file) == 0: return ptrack_bits_for_fork byte_size = os.path.getsize(file + '_ptrack') - npages = byte_size/8192 + npages = int(byte_size/8192) if byte_size % 8192 != 0: print('Ptrack page is not 8k aligned') - sys.exit(1) + exit(1) file = os.open(file + '_ptrack', os.O_RDONLY) @@ -505,9 +774,6 @@ def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): return ptrack_bits_for_fork def check_ptrack_map_sanity(self, node, idx_ptrack): - if node.major_version >= 12: - return - success = True for i in idx_ptrack: # get new size of heap and indexes. size calculated in pages @@ -680,7 +946,7 @@ def check_ptrack_clean(self, idx_dict, size): ) ) - def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True): + def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True, env=None): if not self.probackup_old_path and old_binary: print('PGPROBACKUPBIN_OLD is not set') exit(1) @@ -690,24 +956,27 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur else: binary_path = self.probackup_path + if not env: + env=self.test_env + try: self.cmd = [' '.join(map(str, [binary_path] + command))] if self.verbose: print(self.cmd) if gdb: - return GDBobj([binary_path] + command, self.verbose) + return GDBobj([binary_path] + command, self) if asynchronous: return subprocess.Popen( - self.cmd, + [binary_path] + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.test_env + env=env ) else: self.output = subprocess.check_output( [binary_path] + command, stderr=subprocess.STDOUT, - env=self.test_env + env=env ).decode('utf-8') if command[0] == 'backup' and return_id: # return backup ID @@ -717,9 +986,14 @@ def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, retur else: return self.output except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8'), self.cmd) + raise ProbackupException(e.output.decode('utf-8').replace("\r",""), + self.cmd) + + def run_binary(self, command, asynchronous=False, env=None): + + if not env: + env = self.test_env - def run_binary(self, command, asynchronous=False): if self.verbose: print([' '.join(map(str, command))]) try: @@ -729,13 +1003,13 @@ def run_binary(self, command, asynchronous=False): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self.test_env + env=env ) else: self.output = subprocess.check_output( command, stderr=subprocess.STDOUT, - env=self.test_env + env=env ).decode('utf-8') return self.output except subprocess.CalledProcessError as e: @@ -818,7 +1092,8 @@ def backup_node( self, backup_dir, instance, node, data_dir=False, backup_type='full', datname=False, options=[], asynchronous=False, gdb=False, - old_binary=False, return_id=True, no_remote=False + old_binary=False, return_id=True, no_remote=False, + env=None ): if not node and not data_dir: print('You must provide ether node or data_dir for backup') @@ -851,7 +1126,7 @@ def backup_node( if not old_binary: cmd_list += ['--no-sync'] - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id) + return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) def checkdb_node( self, backup_dir=False, instance=False, data_dir=False, @@ -913,9 +1188,32 @@ def restore_node( return self.run_pb(cmd_list + options, gdb=gdb, old_binary=old_binary) + def catchup_node( + self, + backup_mode, source_pgdata, destination_node, + options = [] + ): + + cmd_list = [ + 'catchup', + '--backup-mode={0}'.format(backup_mode), + '--source-pgdata={0}'.format(source_pgdata), + '--destination-pgdata={0}'.format(destination_node.data_dir) + ] + if self.remote: + cmd_list += ['--remote-proto=ssh', '--remote-host=localhost'] + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(destination_node.logs_dir) + ] + + return self.run_pb(cmd_list + options) + def show_pb( self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False, as_json=True, old_binary=False + options=[], as_text=False, as_json=True, old_binary=False, + env=None ): backup_list = [] @@ -936,7 +1234,7 @@ def show_pb( if as_text: # You should print it when calling as_text=true - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, old_binary=old_binary, env=env) # get show result as list of lines if as_json: @@ -961,7 +1259,7 @@ def show_pb( return backup_list else: show_splitted = self.run_pb( - cmd_list + options, old_binary=old_binary).splitlines() + cmd_list + options, old_binary=old_binary, env=env).splitlines() if instance is not None and backup_id is None: # cut header(ID, Mode, etc) from show as single string header = show_splitted[1:2][0] @@ -1072,8 +1370,8 @@ def show_archive( exit(1) def validate_pb( - self, backup_dir, instance=None, - backup_id=None, options=[], old_binary=False, gdb=False + self, backup_dir, instance=None, backup_id=None, + options=[], old_binary=False, gdb=False, asynchronous=False ): cmd_list = [ @@ -1085,11 +1383,11 @@ def validate_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) def delete_pb( - self, backup_dir, instance, - backup_id=None, options=[], old_binary=False): + self, backup_dir, instance, backup_id=None, + options=[], old_binary=False, gdb=False, asynchronous=False): cmd_list = [ 'delete', '-B', backup_dir @@ -1099,7 +1397,7 @@ def delete_pb( if backup_id: cmd_list += ['-i', backup_id] - return self.run_pb(cmd_list + options, old_binary=old_binary) + return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) def delete_expired( self, backup_dir, instance, options=[], old_binary=False): @@ -1129,8 +1427,9 @@ def get_recovery_conf(self, node): out_dict = {} if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf_path = os.path.join( - node.data_dir, 'probackup_recovery.conf') + recovery_conf_path = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf_path, 'r') as f: + print(f.read()) else: recovery_conf_path = os.path.join(node.data_dir, 'recovery.conf') @@ -1148,7 +1447,8 @@ def get_recovery_conf(self, node): def set_archiving( self, backup_dir, instance, node, replica=False, overwrite=False, compress=True, old_binary=False, - log_level=False, archive_timeout=False): + log_level=False, archive_timeout=False, + custom_archive_command=None): # parse postgresql.auto.conf options = {} @@ -1158,45 +1458,47 @@ def set_archiving( else: options['archive_mode'] = 'on' - if os.name == 'posix': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) - - elif os.name == 'nt': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path.replace("\\","\\\\"), - backup_dir.replace("\\","\\\\"), instance) + if custom_archive_command is None: + if os.name == 'posix': + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path, backup_dir, instance) - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options['archive_command'] += '--remote-proto=ssh ' - options['archive_command'] += '--remote-host=localhost ' + elif os.name == 'nt': + options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( + self.probackup_path.replace("\\","\\\\"), + backup_dir.replace("\\","\\\\"), instance) - if self.archive_compress and compress: - options['archive_command'] += '--compress ' + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options['archive_command'] += '--remote-proto=ssh ' + options['archive_command'] += '--remote-host=localhost ' - if overwrite: - options['archive_command'] += '--overwrite ' + if self.archive_compress and compress: + options['archive_command'] += '--compress ' - options['archive_command'] += '--log-level-console=VERBOSE ' - options['archive_command'] += '-j 5 ' - options['archive_command'] += '--batch-size 10 ' - options['archive_command'] += '--no-sync ' + if overwrite: + options['archive_command'] += '--overwrite ' - if archive_timeout: - options['archive_command'] += '--archive-timeout={0} '.format( - archive_timeout) + options['archive_command'] += '--log-level-console=VERBOSE ' + options['archive_command'] += '-j 5 ' + options['archive_command'] += '--batch-size 10 ' + options['archive_command'] += '--no-sync ' - if os.name == 'posix': - options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' + if archive_timeout: + options['archive_command'] += '--archive-timeout={0} '.format( + archive_timeout) - elif os.name == 'nt': - options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' + if os.name == 'posix': + options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' - if log_level: - options['archive_command'] += ' --log-level-console={0}'.format(log_level) - options['archive_command'] += ' --log-level-file={0} '.format(log_level) + elif os.name == 'nt': + options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' + if log_level: + options['archive_command'] += ' --log-level-console={0}'.format(log_level) + options['archive_command'] += ' --log-level-file={0} '.format(log_level) + else: # custom_archive_command is not None + options['archive_command'] = custom_archive_command self.set_auto_conf(node, options) @@ -1226,7 +1528,9 @@ def get_restore_command(self, backup_dir, instance, node): return restore_command - def set_auto_conf(self, node, options, config='postgresql.auto.conf'): + # rm_options - list of parameter name that should be deleted from current config, + # example: ['wal_keep_segments', 'max_wal_size'] + def set_auto_conf(self, node, options, config='postgresql.auto.conf', rm_options={}): # parse postgresql.auto.conf path = os.path.join(node.data_dir, config) @@ -1254,6 +1558,11 @@ def set_auto_conf(self, node, options, config='postgresql.auto.conf'): var = var.strip() var = var.strip('"') var = var.strip("'") + + # remove options specified in rm_options list + if name in rm_options: + continue + current_options[name] = var for option in options: @@ -1291,10 +1600,6 @@ def set_replica( f.close() config = 'postgresql.auto.conf' - probackup_recovery_path = os.path.join(replica.data_dir, 'probackup_recovery.conf') - if os.path.exists(probackup_recovery_path): - if os.stat(probackup_recovery_path).st_size > 0: - config = 'probackup_recovery.conf' if not log_shipping: self.set_auto_conf( @@ -1393,7 +1698,7 @@ def version_to_num(self, version): parts.append('0') num = 0 for part in parts: - num = num * 100 + int(re.sub("[^\d]", "", part)) + num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num def switch_wal_segment(self, node): @@ -1405,7 +1710,7 @@ def switch_wal_segment(self, node): """ if isinstance(node, testgres.PostgresNode): if self.version_to_num( - node.safe_psql('postgres', 'show server_version') + node.safe_psql('postgres', 'show server_version').decode('utf-8') ) >= self.version_to_num('10.0'): node.safe_psql('postgres', 'select pg_switch_wal()') else: @@ -1422,10 +1727,11 @@ def switch_wal_segment(self, node): def wait_until_replica_catch_with_master(self, master, replica): - if self.version_to_num( - master.safe_psql( - 'postgres', - 'show server_version')) >= self.version_to_num('10.0'): + version = master.safe_psql( + 'postgres', + 'show server_version').decode('utf-8').rstrip() + + if self.version_to_num(version) >= self.version_to_num('10.0'): master_function = 'pg_catalog.pg_current_wal_lsn()' replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' else: @@ -1434,7 +1740,7 @@ def wait_until_replica_catch_with_master(self, master, replica): lsn = master.safe_psql( 'postgres', - 'SELECT {0}'.format(master_function)).rstrip() + 'SELECT {0}'.format(master_function)).decode('utf-8').rstrip() # Wait until replica catch up with master replica.poll_query_until( @@ -1445,18 +1751,18 @@ def get_version(self, node): return self.version_to_num( testgres.get_pg_config()['VERSION'].split(" ")[1]) + def get_ptrack_version(self, node): + version = node.safe_psql( + "postgres", + "SELECT extversion " + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() + return self.version_to_num(version) + def get_bin_path(self, binary): return testgres.get_bin_path(binary) - def del_test_dir(self, module_name, fname, nodes=[]): + def del_test_dir(self, module_name, fname): """ Del testdir and optimistically try to del module dir""" - try: - testgres.clean_all() - except: - pass - - for node in nodes: - node.stop() shutil.rmtree( os.path.join( @@ -1466,10 +1772,6 @@ def del_test_dir(self, module_name, fname, nodes=[]): ), ignore_errors=True ) - try: - os.rmdir(os.path.join(self.tmp_path, module_name)) - except: - pass def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): """ return dict with directory content. " @@ -1484,7 +1786,8 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): 'backup_label', 'tablespace_map', 'recovery.conf', 'ptrack_control', 'ptrack_init', 'pg_control', 'probackup_recovery.conf', 'recovery.signal', - 'standby.signal', 'ptrack.map', 'ptrack.map.mmap' + 'standby.signal', 'ptrack.map', 'ptrack.map.mmap', + 'ptrack.map.tmp', 'recovery.done','backup_label.old' ] if exclude_dirs: @@ -1507,191 +1810,225 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): file_fullpath = os.path.join(root, file) file_relpath = os.path.relpath(file_fullpath, pgdata) - directory_dict['files'][file_relpath] = {'is_datafile': False} - directory_dict['files'][file_relpath]['md5'] = hashlib.md5( - open(file_fullpath, 'rb').read()).hexdigest() + cfile = ContentFile(file.isdigit()) + directory_dict['files'][file_relpath] = cfile + with open(file_fullpath, 'rb') as f: + # truncate cfm's content's zero tail + if file_relpath.endswith('.cfm'): + content = f.read() + zero64 = b"\x00"*64 + l = len(content) + while l > 64: + s = (l - 1) & ~63 + if content[s:l] != zero64[:l-s]: + break + l = s + content = content[:l] + digest = hashlib.md5(content) + else: + digest = hashlib.md5() + while True: + b = f.read(64*1024) + if not b: break + digest.update(b) + cfile.md5 = digest.hexdigest() # crappy algorithm - if file.isdigit(): - directory_dict['files'][file_relpath]['is_datafile'] = True + if cfile.is_datafile: size_in_pages = os.path.getsize(file_fullpath)/8192 - directory_dict['files'][file_relpath][ - 'md5_per_page'] = self.get_md5_per_page_for_fork( + cfile.md5_per_page = self.get_md5_per_page_for_fork( file_fullpath, size_in_pages ) - for root, dirs, files in os.walk(pgdata, topdown=False, followlinks=True): for directory in dirs: directory_path = os.path.join(root, directory) directory_relpath = os.path.relpath(directory_path, pgdata) - - found = False - for d in dirs_to_ignore: - if d in directory_relpath: - found = True - break - - # check if directory already here as part of larger directory - if not found: - for d in directory_dict['dirs']: - # print("OLD dir {0}".format(d)) - if directory_relpath in d: - found = True - break - - if not found: - directory_dict['dirs'][directory_relpath] = {} + parent = os.path.dirname(directory_relpath) + if parent in directory_dict['dirs']: + del directory_dict['dirs'][parent] + directory_dict['dirs'][directory_relpath] = ContentDir() # get permissions for every file and directory - for file in directory_dict['dirs']: - full_path = os.path.join(pgdata, file) - directory_dict['dirs'][file]['mode'] = os.stat( - full_path).st_mode + for dir, cdir in directory_dict['dirs'].items(): + full_path = os.path.join(pgdata, dir) + cdir.mode = os.stat(full_path).st_mode - for file in directory_dict['files']: + for file, cfile in directory_dict['files'].items(): full_path = os.path.join(pgdata, file) - directory_dict['files'][file]['mode'] = os.stat( - full_path).st_mode + cfile.mode = os.stat(full_path).st_mode return directory_dict - def compare_pgdata(self, original_pgdata, restored_pgdata): - """ return dict with directory content. DO IT BEFORE RECOVERY""" + def get_known_bugs_comparision_exclusion_dict(self, node): + """ get dict of known datafiles difference, that can be used in compare_pgdata() """ + comparision_exclusion_dict = dict() + + # bug in spgist metapage update (PGPRO-5707) + spgist_filelist = node.safe_psql( + "postgres", + "SELECT pg_catalog.pg_relation_filepath(pg_class.oid) " + "FROM pg_am, pg_class " + "WHERE pg_am.amname = 'spgist' " + "AND pg_class.relam = pg_am.oid" + ).decode('utf-8').rstrip().splitlines() + for filename in spgist_filelist: + comparision_exclusion_dict[filename] = set([0]) + + return comparision_exclusion_dict + + + def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict()): + """ + return dict with directory content. DO IT BEFORE RECOVERY + exclusion_dict is used for exclude files (and it block_no) from comparision + it is a dict with relative filenames as keys and set of block numbers as values + """ fail = False error_message = 'Restored PGDATA is not equal to original!\n' # Compare directories - for directory in restored_pgdata['dirs']: - if directory not in original_pgdata['dirs']: - fail = True - error_message += '\nDirectory was not present' - error_message += ' in original PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory)) - else: - if ( - restored_pgdata['dirs'][directory]['mode'] != - original_pgdata['dirs'][directory]['mode'] - ): - fail = True - error_message += '\nDir permissions mismatch:\n' - error_message += ' Dir old: {0} Permissions: {1}\n'.format( - os.path.join(original_pgdata['pgdata'], directory), - original_pgdata['dirs'][directory]['mode']) - error_message += ' Dir new: {0} Permissions: {1}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory), - restored_pgdata['dirs'][directory]['mode']) - - for directory in original_pgdata['dirs']: - if directory not in restored_pgdata['dirs']: + restored_dirs = set(restored_pgdata['dirs']) + original_dirs = set(original_pgdata['dirs']) + + for directory in sorted(restored_dirs - original_dirs): + fail = True + error_message += '\nDirectory was not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in sorted(original_dirs - restored_dirs): + fail = True + error_message += '\nDirectory dissappeared' + error_message += ' in restored PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory)) + + for directory in sorted(original_dirs & restored_dirs): + original = original_pgdata['dirs'][directory] + restored = restored_pgdata['dirs'][directory] + if original.mode != restored.mode: fail = True - error_message += '\nDirectory dissappeared' - error_message += ' in restored PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], directory)) - - for file in restored_pgdata['files']: + error_message += '\nDir permissions mismatch:\n' + error_message += ' Dir old: {0} Permissions: {1}\n'.format( + os.path.join(original_pgdata['pgdata'], directory), + original.mode) + error_message += ' Dir new: {0} Permissions: {1}\n'.format( + os.path.join(restored_pgdata['pgdata'], directory), + restored.mode) + + restored_files = set(restored_pgdata['files']) + original_files = set(original_pgdata['files']) + + for file in sorted(restored_files - original_files): # File is present in RESTORED PGDATA # but not present in ORIGINAL # only backup_label is allowed - if file not in original_pgdata['files']: - fail = True - error_message += '\nFile is not present' - error_message += ' in original PGDATA: {0}\n'.format( - os.path.join(restored_pgdata['pgdata'], file)) - - for file in original_pgdata['files']: - if file in restored_pgdata['files']: - - if ( - restored_pgdata['files'][file]['mode'] != - original_pgdata['files'][file]['mode'] - ): - fail = True - error_message += '\nFile permissions mismatch:\n' - error_message += ' File_old: {0} Permissions: {1}\n'.format( - os.path.join(original_pgdata['pgdata'], file), - original_pgdata['files'][file]['mode']) - error_message += ' File_new: {0} Permissions: {1}\n'.format( - os.path.join(restored_pgdata['pgdata'], file), - restored_pgdata['files'][file]['mode']) + fail = True + error_message += '\nFile is not present' + error_message += ' in original PGDATA: {0}\n'.format( + os.path.join(restored_pgdata['pgdata'], file)) + + for file in sorted(original_files - restored_files): + error_message += ( + '\nFile disappearance.\n ' + 'File: {0}\n').format( + os.path.join(restored_pgdata['pgdata'], file) + ) + fail = True - if ( - original_pgdata['files'][file]['md5'] != - restored_pgdata['files'][file]['md5'] - ): + for file in sorted(original_files & restored_files): + original = original_pgdata['files'][file] + restored = restored_pgdata['files'][file] + if restored.mode != original.mode: + fail = True + error_message += '\nFile permissions mismatch:\n' + error_message += ' File_old: {0} Permissions: {1:o}\n'.format( + os.path.join(original_pgdata['pgdata'], file), + original.mode) + error_message += ' File_new: {0} Permissions: {1:o}\n'.format( + os.path.join(restored_pgdata['pgdata'], file), + restored.mode) + + if original.md5 != restored.md5: + if file not in exclusion_dict: fail = True error_message += ( - '\nFile Checksumm mismatch.\n' - 'File_old: {0}\nChecksumm_old: {1}\n' - 'File_new: {2}\nChecksumm_new: {3}\n').format( + '\nFile Checksum mismatch.\n' + 'File_old: {0}\nChecksum_old: {1}\n' + 'File_new: {2}\nChecksum_new: {3}\n').format( os.path.join(original_pgdata['pgdata'], file), - original_pgdata['files'][file]['md5'], + original.md5, os.path.join(restored_pgdata['pgdata'], file), - restored_pgdata['files'][file]['md5'] + restored.md5 ) - if original_pgdata['files'][file]['is_datafile']: - for page in original_pgdata['files'][file]['md5_per_page']: - if page not in restored_pgdata['files'][file]['md5_per_page']: - error_message += ( - '\n Page {0} dissappeared.\n ' - 'File: {1}\n').format( - page, - os.path.join( - restored_pgdata['pgdata'], - file - ) - ) - continue - - if original_pgdata['files'][file][ - 'md5_per_page'][page] != restored_pgdata[ - 'files'][file]['md5_per_page'][page]: - error_message += ( - '\n Page checksumm mismatch: {0}\n ' - ' PAGE Checksumm_old: {1}\n ' - ' PAGE Checksumm_new: {2}\n ' - ' File: {3}\n' - ).format( - page, - original_pgdata['files'][file][ - 'md5_per_page'][page], - restored_pgdata['files'][file][ - 'md5_per_page'][page], - os.path.join( - restored_pgdata['pgdata'], file) - ) - for page in restored_pgdata['files'][file]['md5_per_page']: - if page not in original_pgdata['files'][file]['md5_per_page']: - error_message += '\n Extra page {0}\n File: {1}\n'.format( - page, - os.path.join( - restored_pgdata['pgdata'], file)) + if not original.is_datafile: + continue - else: - error_message += ( - '\nFile disappearance.\n ' - 'File: {0}\n').format( - os.path.join(restored_pgdata['pgdata'], file) + original_pages = set(original.md5_per_page) + restored_pages = set(restored.md5_per_page) + + for page in sorted(original_pages - restored_pages): + error_message += '\n Page {0} dissappeared.\n File: {1}\n'.format( + page, + os.path.join(restored_pgdata['pgdata'], file) ) - fail = True + + + for page in sorted(restored_pages - original_pages): + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, + os.path.join(restored_pgdata['pgdata'], file)) + + for page in sorted(original_pages & restored_pages): + if file in exclusion_dict and page in exclusion_dict[file]: + continue + + if original.md5_per_page[page] != restored.md5_per_page[page]: + fail = True + error_message += ( + '\n Page checksum mismatch: {0}\n ' + ' PAGE Checksum_old: {1}\n ' + ' PAGE Checksum_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original.md5_per_page[page], + restored.md5_per_page[page], + os.path.join( + restored_pgdata['pgdata'], file) + ) + self.assertFalse(fail, error_message) def gdb_attach(self, pid): - return GDBobj([str(pid)], self.verbose, attach=True) + return GDBobj([str(pid)], self, attach=True) + + def _check_gdb_flag_or_skip_test(self): + if not self.gdb: + self.skipTest( + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) class GdbException(Exception): - def __init__(self, message=False): + def __init__(self, message="False"): self.message = message def __str__(self): return '\n ERROR: {0}\n'.format(repr(self.message)) -class GDBobj(ProbackupTest): - def __init__(self, cmd, verbose, attach=False): - self.verbose = verbose +class GDBobj: + def __init__(self, cmd, env, attach=False): + self.verbose = env.verbose + self.output = '' + # Check gdb flag is set up + if not env.gdb: + raise GdbException("No `PGPROBACKUP_GDB=on` is set, " + "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " + "and be skipped") # Check gdb presense try: gdb_version, _ = subprocess.Popen( @@ -1714,7 +2051,7 @@ def __init__(self, cmd, verbose, attach=False): # Get version gdb_version_number = re.search( - b"^GNU gdb [^\d]*(\d+)\.(\d)", + br"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) self.major_version = int(gdb_version_number.group(1)) self.minor_version = int(gdb_version_number.group(2)) @@ -1728,14 +2065,13 @@ def __init__(self, cmd, verbose, attach=False): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0, - universal_newlines=True + text=True, + errors='replace', ) self.gdb_pid = self.proc.pid - # discard data from pipe, - # is there a way to do it a less derpy way? while True: - line = self.proc.stdout.readline() + line = self.get_line() if 'No such process' in line: raise GdbException(line) @@ -1745,6 +2081,15 @@ def __init__(self, cmd, verbose, attach=False): else: break + def get_line(self): + line = self.proc.stdout.readline() + self.output += line + return line + + def kill(self): + self.proc.kill() + self.proc.wait() + def set_breakpoint(self, location): result = self._execute('break ' + location) @@ -1862,16 +2207,17 @@ def continue_execution_until_break(self, ignore_count=0): 'Failed to continue execution until break.\n') def stopped_in_breakpoint(self): - output = [] while True: - line = self.proc.stdout.readline() - output += [line] + line = self.get_line() if self.verbose: print(line) if line.startswith('*stopped,reason="breakpoint-hit"'): return True return False + def quit(self): + self.proc.terminate() + # use for breakpoint, run, continue def _execute(self, cmd, running=True): output = [] @@ -1882,7 +2228,7 @@ def _execute(self, cmd, running=True): # look for command we just send while True: - line = self.proc.stdout.readline() + line = self.get_line() if self.verbose: print(repr(line)) @@ -1892,7 +2238,7 @@ def _execute(self, cmd, running=True): break while True: - line = self.proc.stdout.readline() + line = self.get_line() output += [line] if self.verbose: print(repr(line)) @@ -1904,3 +2250,10 @@ def _execute(self, cmd, running=True): # if running and line.startswith('*running'): break return output +class ContentFile(object): + __slots__ = ('is_datafile', 'mode', 'md5', 'md5_per_page') + def __init__(self, is_datafile: bool): + self.is_datafile = is_datafile + +class ContentDir(object): + __slots__ = ('mode') diff --git a/tests/incr_restore.py b/tests/incr_restore_test.py similarity index 58% rename from tests/incr_restore.py rename to tests/incr_restore_test.py index 9caa479c0..6a2164098 100644 --- a/tests/incr_restore.py +++ b/tests/incr_restore_test.py @@ -9,24 +9,20 @@ import hashlib import shutil import json -from testgres import QueryException - - -module_name = 'incr_restore' - +from testgres import QueryException, StartNodeException +import stat +from stat import S_ISDIR class IncrRestoreTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_basic_incr_restore(self): """incremental restore in CHECKSUM mode""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -70,25 +66,66 @@ def test_basic_incr_restore(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_basic_incr_restore_into_missing_directory(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=10) + + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + pgbench.wait() + pgbench.stdout.close() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_checksum_corruption_detection(self): - """recovery to target timeline""" - fname = self.id().split('.')[3] + """ + """ node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -127,24 +164,22 @@ def test_checksum_corruption_detection(self): node.stop() self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) + backup_dir, 'node', node, + options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -168,25 +203,21 @@ def test_incr_restore_with_tablespace(self): self.restore_node( backup_dir, 'node', node, options=[ - "-j", "4", "--incremental-mode=checksum", + "-j", "4", "--incremental-mode=checksum", "--force", "-T{0}={1}".format(tblspace, some_directory)]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_1(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -232,65 +263,356 @@ def test_incr_restore_with_tablespace_1(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_restore_with_tablespace_2(self): """ If "--tablespace-mapping" option is used with incremental restore, then new directory must be empty. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + + # fill node1 with data + out = self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '--force']) + + self.assertIn("WARNING: Backup catalog was initialized for system id", out) + tblspace = self.get_tblspace_path(node, 'tblspace') self.create_tblspace_in_node(node, 'tblspace') - node.pgbench_init(scale=10, tablespace='tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') - self.backup_node(backup_dir, 'node', node, options=['--stream']) + node.safe_psql( + 'postgres', + 'vacuum') - pgdata = self.pgdata_content(node.data_dir) + self.backup_node(backup_dir, 'node', node, backup_type='delta', options=['--stream']) - node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + pgdata = self.pgdata_content(node.data_dir) - node_1.cleanup() + try: + self.restore_node( + backup_dir, 'node', node, + data_dir=node_1.data_dir, + options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped directory is not empty.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.restore_node( + out = self.restore_node( backup_dir, 'node', node, data_dir=node_1.data_dir, - options=['--incremental-mode=checksum']) + options=[ + '--force', '--incremental-mode=checksum', + '-T{0}={1}'.format(tblspace, tblspace)]) + + pgdata_restored = self.pgdata_content(node_1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_3(self): + """ + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + + self.create_tblspace_in_node(node, 'tblspace2') + node.pgbench_init(scale=10, tablespace='tblspace2') + + node.stop() self.restore_node( backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) + options=[ + "-j", "4", + "--incremental-mode=checksum"]) - pgdata_restored = self.pgdata_content(node_1.data_dir) + pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_4(self): + """ + Check that system ID mismatch is detected, + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + + # take backup of node1 with tblspace1 + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.drop_tblspace(node, 'tblspace1') + node.cleanup() + + # recreate node + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace1') + node.pgbench_init(scale=10, tablespace='tblspace1') + node.stop() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because destination directory has wrong system id.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'WARNING: Backup catalog was initialized for system id', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.assertIn( + 'ERROR: Incremental restore is not allowed', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.expectedFailure + @unittest.skip("skip") + def test_incr_restore_with_tablespace_5(self): + """ + More complicated case, we restore backup + with tablespace, which we remap into directory + with some old content, that belongs to an instance + with different system id. + """ + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node1) + node1.slow_start() + + self.create_tblspace_in_node(node1, 'tblspace') + node1.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node1 with tblspace + self.backup_node(backup_dir, 'node', node1, options=['--stream']) + pgdata = self.pgdata_content(node1.data_dir) + + node1.stop() + + # recreate node + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2'), + set_replication=True, + initdb_params=['--data-checksums']) + node2.slow_start() + + self.create_tblspace_in_node(node2, 'tblspace') + node2.pgbench_init(scale=10, tablespace='tblspace') + node2.stop() + + tblspc1_path = self.get_tblspace_path(node1, 'tblspace') + tblspc2_path = self.get_tblspace_path(node2, 'tblspace') + + out = self.restore_node( + backup_dir, 'node', node1, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum", + "-T{0}={1}".format(tblspc1_path, tblspc2_path)]) + + # check that tblspc1_path is empty + self.assertFalse( + os.listdir(tblspc1_path), + "Dir is not empty: '{0}'".format(tblspc1_path)) + + pgdata_restored = self.pgdata_content(node1.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_6(self): + """ + Empty pgdata, not empty tablespace + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=10, tablespace='tblspace') + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + try: + self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", + "--incremental-mode=checksum"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because there is running postmaster " + "process in destination directory.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: PGDATA is empty, but tablespace destination is not', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--force", + "--incremental-mode=checksum"]) + + self.assertIn( + "INFO: Destination directory and tablespace directories are empty, " + "disable incremental restore", out) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_incr_restore_with_tablespace_7(self): + """ + Restore backup without tablespace into + PGDATA with tablespace. + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + # take backup of node with tblspace + self.backup_node(backup_dir, 'node', node, options=['--stream']) + pgdata = self.pgdata_content(node.data_dir) + + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=5, tablespace='tblspace') + node.stop() + +# try: +# self.restore_node( +# backup_dir, 'node', node, +# options=[ +# "-j", "4", +# "--incremental-mode=checksum"]) +# # we should die here because exception is what we expect to happen +# self.assertEqual( +# 1, 0, +# "Expecting Error because there is running postmaster " +# "process in destination directory.\n " +# "Output: {0} \n CMD: {1}".format( +# repr(self.output), self.cmd)) +# except ProbackupException as e: +# self.assertIn( +# 'ERROR: PGDATA is empty, but tablespace destination is not', +# e.message, +# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( +# repr(e.message), self.cmd)) + + out = self.restore_node( + backup_dir, 'node', node, + options=[ + "-j", "4", "--incremental-mode=checksum"]) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_basic_incr_restore_sanity(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -315,13 +637,13 @@ def test_basic_incr_restore_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Incremental restore is impossible', + 'ERROR: Incremental restore is not allowed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) try: self.restore_node( @@ -340,14 +662,11 @@ def test_basic_incr_restore_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) self.assertIn( - 'ERROR: Incremental restore is impossible', + 'ERROR: Incremental restore is not allowed', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_incr_checksum_restore(self): """ @@ -356,13 +675,12 @@ def test_incr_checksum_restore(self): X - is instance, we want to return it to C state. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -381,7 +699,7 @@ def test_incr_checksum_restore(self): xid = node.safe_psql( 'postgres', - 'select txid_current()').rstrip() + 'select txid_current()').decode('utf-8').rstrip() # --A-----B--------X pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) @@ -389,7 +707,7 @@ def test_incr_checksum_restore(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -423,9 +741,9 @@ def test_incr_checksum_restore(self): pgdata = self.pgdata_content(node_1.data_dir) - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -434,9 +752,6 @@ def test_incr_checksum_restore(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_1]) - # @unittest.skip("skip") def test_incr_lsn_restore(self): @@ -446,13 +761,12 @@ def test_incr_lsn_restore(self): X - is instance, we want to return it to C state. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -471,7 +785,7 @@ def test_incr_lsn_restore(self): xid = node.safe_psql( 'postgres', - 'select txid_current()').rstrip() + 'select txid_current()').decode('utf-8').rstrip() # --A-----B--------X pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) @@ -479,7 +793,7 @@ def test_incr_lsn_restore(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -513,8 +827,8 @@ def test_incr_lsn_restore(self): pgdata = self.pgdata_content(node_1.data_dir) - print(self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"])) + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -523,9 +837,6 @@ def test_incr_lsn_restore(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_1]) - # @unittest.skip("skip") def test_incr_lsn_sanity(self): """ @@ -535,13 +846,12 @@ def test_incr_lsn_sanity(self): X - is instance, we want to return it to state B. fail is expected behaviour in case of lsn restore. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -551,7 +861,7 @@ def test_incr_lsn_sanity(self): node.pgbench_init(scale=10) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -592,9 +902,6 @@ def test_incr_lsn_sanity(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname, [node_1]) - # @unittest.skip("skip") def test_incr_checksum_sanity(self): """ @@ -603,13 +910,11 @@ def test_incr_checksum_sanity(self): X - is instance, we want to return it to state B. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -619,7 +924,7 @@ def test_incr_checksum_sanity(self): node.pgbench_init(scale=20) node_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_1')) node_1.cleanup() self.restore_node( @@ -651,22 +956,17 @@ def test_incr_checksum_sanity(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname, [node_1]) - - # @unittest.skip("skip") def test_incr_checksum_corruption_detection(self): """ check that corrupted page got detected and replaced """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), # initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -683,7 +983,7 @@ def test_incr_checksum_corruption_detection(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() @@ -702,29 +1002,25 @@ def test_incr_checksum_corruption_detection(self): f.flush() f.close - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=checksum"])) + options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_incr_lsn_corruption_detection(self): """ check that corrupted page got detected and replaced """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -741,7 +1037,7 @@ def test_incr_lsn_corruption_detection(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() @@ -768,21 +1064,16 @@ def test_incr_lsn_corruption_detection(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -832,29 +1123,24 @@ def test_incr_restore_multiple_external(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=checksum', '--log-level-console=VERBOSE'])) + options=["-j", "4", '--incremental-mode=checksum']) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_restore_multiple_external(self): """check that cmdline has priority over config""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -904,30 +1190,26 @@ def test_incr_lsn_restore_multiple_external(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, - options=["-j", "4", '--incremental-mode=lsn'])) + options=["-j", "4", '--incremental-mode=lsn']) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_restore_backward(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on', 'hot_standby': 'on'}) + pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -964,11 +1246,13 @@ def test_incr_lsn_restore_backward(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=[ - "-j", "4", '--incremental-mode=lsn', '--log-level-file=VERBOSE', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(full_pgdata, pgdata_restored) @@ -1007,33 +1291,30 @@ def test_incr_lsn_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=delta_id, options=[ - "-j", "4", '--incremental-mode=lsn', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_checksum_restore_backward(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1070,11 +1351,13 @@ def test_incr_checksum_restore_backward(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(full_pgdata, pgdata_restored) @@ -1082,11 +1365,13 @@ def test_incr_checksum_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=page_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(page_pgdata, pgdata_restored) @@ -1094,32 +1379,29 @@ def test_incr_checksum_restore_backward(self): node.slow_start(replica=True) node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=delta_id, options=[ - "-j", "4", '--incremental-mode=checksum', - '--recovery-target=immediate', '--recovery-target-action=pause'])) + "-j", "4", + '--incremental-mode=checksum', + '--recovery-target=immediate', + '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(delta_pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_make_replica_via_incr_checksum_restore(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1128,7 +1410,7 @@ def test_make_replica_via_incr_checksum_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.pgbench_init(scale=20) @@ -1165,9 +1447,9 @@ def test_make_replica_via_incr_checksum_restore(self): data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - print(self.restore_node( + self.restore_node( backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=checksum'])) + options=['-R', '--incremental-mode=checksum']) self.set_replica(new_master, old_master, synchronous=True) @@ -1176,23 +1458,18 @@ def test_make_replica_via_incr_checksum_restore(self): pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname, [new_master, old_master]) - # @unittest.skip("skip") def test_make_replica_via_incr_lsn_restore(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1201,7 +1478,7 @@ def test_make_replica_via_incr_lsn_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() master.pgbench_init(scale=20) @@ -1238,9 +1515,9 @@ def test_make_replica_via_incr_lsn_restore(self): data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - print(self.restore_node( + self.restore_node( backup_dir, 'node', old_master, data_dir=old_master.data_dir, - options=['-R', '--incremental-mode=lsn'])) + options=['-R', '--incremental-mode=lsn']) self.set_replica(new_master, old_master, synchronous=True) @@ -1249,23 +1526,16 @@ def test_make_replica_via_incr_lsn_restore(self): pgbench = new_master.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname, [new_master, old_master]) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_checksum_long_xact(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, -# initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1319,26 +1589,22 @@ def test_incr_checksum_long_xact(self): self.assertEqual( node.safe_psql( 'postgres', - 'select count(*) from t1').rstrip(), + 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure + # This test will pass with Enterprise + # because it has checksums enabled by default + @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, -# initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1385,9 +1651,9 @@ def test_incr_lsn_long_xact_1(self): node.stop() try: - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn'])) + options=["-j", "4", '--incremental-mode=lsn']) # we should die here because exception is what we expect to happen self.assertEqual( 1, 0, @@ -1402,25 +1668,20 @@ def test_incr_lsn_long_xact_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_long_xact_2(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'off', 'wal_log_hints': 'off'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1487,31 +1748,26 @@ def test_incr_lsn_long_xact_2(self): self.assertEqual( node.safe_psql( 'postgres', - 'select count(*) from t1').rstrip(), + 'select count(*) from t1').decode('utf-8').rstrip(), '1') - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_zero_size_file_checksum(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w", 0) as f: + with open(fullpath, "w+b", 0) as f: f.flush() f.close @@ -1543,9 +1799,9 @@ def test_incr_restore_zero_size_file_checksum(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum'])) + options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) @@ -1564,28 +1820,23 @@ def test_incr_restore_zero_size_file_checksum(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_restore_zero_size_file_lsn(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') - with open(fullpath, "w", 0) as f: + with open(fullpath, "w+b", 0) as f: f.flush() f.close @@ -1617,9 +1868,9 @@ def test_incr_restore_zero_size_file_lsn(self): node.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node, backup_id=id1, - options=["-j", "4", '-I', 'checksum'])) + options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) @@ -1644,15 +1895,11 @@ def test_incr_restore_zero_size_file_lsn(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_incremental_partial_restore_exclude_checksum(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1691,11 +1938,11 @@ def test_incremental_partial_restore_exclude_checksum(self): # restore FULL backup into second node2 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() # restore some data into node2 @@ -1711,12 +1958,14 @@ def test_incremental_partial_restore_exclude_checksum(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore backup into node2 - print(self.restore_node( + self.restore_node( backup_dir, 'node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "checksum"])) + "-I", "checksum", + "--destroy-all-other-dbs", + ]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -1749,15 +1998,11 @@ def test_incremental_partial_restore_exclude_checksum(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node2]) - def test_incremental_partial_restore_exclude_lsn(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1798,11 +2043,11 @@ def test_incremental_partial_restore_exclude_lsn(self): # restore FULL backup into second node2 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() # restore some data into node2 @@ -1821,12 +2066,14 @@ def test_incremental_partial_restore_exclude_lsn(self): node2.port = node.port node2.slow_start() node2.stop() - print(self.restore_node( + self.restore_node( backup_dir, 'node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", - "-I", "lsn"])) + "-I", "lsn", + "--destroy-all-other-dbs", + ]) pgdata2 = self.pgdata_content(node2.data_dir) @@ -1859,15 +2106,11 @@ def test_incremental_partial_restore_exclude_lsn(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname, [node2]) - def test_incremental_partial_restore_exclude_tablespace_checksum(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1913,13 +2156,13 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): # node1 node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1')) + base_dir=os.path.join(self.module_name, self.fname, 'node1')) node1.cleanup() node1_tablespace = self.get_tblspace_path(node1, 'somedata') # node2 node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() node2_tablespace = self.get_tblspace_path(node2, 'somedata') @@ -1939,24 +2182,47 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): "-T", "{0}={1}".format( node_tablespace, node1_tablespace)]) -# with open(os.path.join(node1_tablespace, "hello"), "w") as f: -# f.close() pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore into node2 + try: + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace), + "--destroy-all-other-dbs"]) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because remapped tablespace contain old data .\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Remapped tablespace destination is not empty:', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + self.restore_node( backup_dir, 'node', node2, options=[ - "-I", "checksum", + "-I", "checksum", "--force", "--db-exclude=db1", "--db-exclude=db5", "-T", "{0}={1}".format( - node_tablespace, node2_tablespace)]) + node_tablespace, node2_tablespace), + "--destroy-all-other-dbs", + ]) + pgdata2 = self.pgdata_content(node2.data_dir) self.compare_pgdata(pgdata1, pgdata2) - self.set_auto_conf(node2, {'port': node2.port}) node2.slow_start() @@ -1983,8 +2249,384 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname, [node2]) + def test_incremental_partial_restore_deny(self): + """ + Do now allow partial incremental restore into non-empty PGDATA + becase we can't limit WAL replay to a single database. + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + for i in range(1, 3): + node.safe_psql('postgres', f'CREATE database db{i}') + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + pgdata = self.pgdata_content(node.data_dir) + + try: + self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) + self.fail("incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Postmaster is running.", e.message) + + node.safe_psql('db2', 'create table x (id int)') + node.safe_psql('db2', 'insert into x values (42)') + + node.stop() + + try: + self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) + self.fail("because incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + + node.slow_start() + value = node.execute('db2', 'select * from x')[0][0] + self.assertEqual(42, value) + + def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self): + """ + Do now allow partial incremental restore into non-empty PGDATA + becase we can't limit WAL replay to a single database. + (case of tablespaces) + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + node_tablespace = self.get_tblspace_path(node, 'somedata') + + tbl_oid = node.safe_psql( + 'postgres', + "SELECT oid " + "FROM pg_tablespace " + "WHERE spcname = 'somedata'").rstrip() + + for i in range(1, 10, 1): + node.safe_psql( + 'postgres', + 'CREATE database db{0} tablespace somedata'.format(i)) + + db_list_raw = node.safe_psql( + 'postgres', + 'SELECT to_json(a) ' + 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + + db_list_splitted = db_list_raw.splitlines() + + db_list = {} + for line in db_list_splitted: + line = json.loads(line) + db_list[line['datname']] = line['oid'] + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # node2 + node2 = self.make_simple_node('node2') + node2.cleanup() + node2_tablespace = self.get_tblspace_path(node2, 'somedata') + + # in node2 restore full backup + self.restore_node( + backup_dir, 'node', + node2, options=[ + "-T", f"{node_tablespace}={node2_tablespace}"]) + + # partial incremental restore into node2 + try: + self.restore_node(backup_dir, 'node', node2, + options=["-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"]) + self.fail("remapped tablespace contain old data") + except ProbackupException as e: + pass + + try: + self.restore_node(backup_dir, 'node', node2, + options=[ + "-I", "checksum", "--force", + "--db-exclude=db1", "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"]) + self.fail("incremental partial restore is not allowed") + except ProbackupException as e: + self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + + def test_incremental_pg_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + # in node1 restore full backup + self.restore_node(backup_dir, 'node', node1) + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + pgbench = node1.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node1.stop() + + # incremental restore into node1 + self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum"]) + + self.set_auto_conf(node1, {'port': node1.port}) + node1.slow_start() + + node1.safe_psql( + 'postgres', + 'select 1') # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn -# incremental restore + partial restore. + + # @unittest.skip("skip") + def test_incr_restore_issue_313(self): + """ + Check that failed incremental restore can be restarted + """ + self._check_gdb_flag_or_skip_test + node = self.make_simple_node('node', + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale = 50) + + full_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + last_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.restore_node(backup_dir, 'node', node, backup_id=full_backup_id) + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', last_backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and \ + not stat.S_ISDIR(int(filelist[file]['mode'])) and \ + not filelist[file]['size'] == '0' and \ + file != 'database_map': + count += 1 + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 1) + gdb.quit() + + bak_file = os.path.join(node.data_dir, 'global', 'pg_control.pbk.bak') + self.assertTrue( + os.path.exists(bak_file), + "pg_control bak File should not exist: {0}".format(bak_file)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + if self.pg_config_version >= 120000: + self.assertIn( + "PANIC: could not read file \"global/pg_control\"", + f.read()) + else: + self.assertIn( + "PANIC: could not read from control file", + f.read()) + self.restore_node(backup_dir, 'node', node, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + node.slow_start() + self.compare_pgdata(pgdata, self.pgdata_content(node.data_dir)) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_checksum(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=checksum", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_lsn(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'wal_log_hints': 'on'}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.restore_node( + backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/init.py b/tests/init_test.py similarity index 81% rename from tests/init.py rename to tests/init_test.py index f5715d249..4e000c78f 100644 --- a/tests/init.py +++ b/tests/init_test.py @@ -4,18 +4,14 @@ import shutil -module_name = 'init' - - class InitTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_success(self): """Success normal init""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.assertEqual( dir_files(backup_dir), @@ -60,19 +56,16 @@ def test_success(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", + "ERROR: No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\ncommand line option --pgdata (-D)", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_already_exist(self): """Failure with backup catalog already existed""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) try: self.show_pb(backup_dir, 'node') @@ -84,15 +77,11 @@ def test_already_exist(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_abs_path(self): """failure with backup catalog should be given as absolute path""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) try: self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( @@ -103,18 +92,14 @@ def test_abs_path(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_add_instance_idempotence(self): """ https://github.com/postgrespro/pg_probackup/issues/219 """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node')) + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -152,6 +137,3 @@ def test_add_instance_idempotence(self): e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/locking.py b/tests/locking_test.py similarity index 74% rename from tests/locking.py rename to tests/locking_test.py index 2da2415ea..5367c2610 100644 --- a/tests/locking.py +++ b/tests/locking_test.py @@ -4,9 +4,6 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'locking' - - class LockingTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -17,12 +14,13 @@ def test_locking_running_validate_1(self): run validate, expect it to successfully executed, concurrent RUNNING backup with pid file and active process is legal """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -50,7 +48,7 @@ def test_locking_running_validate_1(self): backup_id = self.show_pb(backup_dir, 'node')[1]['id'] self.assertIn( - "is using backup {0} and still is running".format(backup_id), + "is using backup {0}, and is still running".format(backup_id), validate_output, '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) @@ -61,7 +59,7 @@ def test_locking_running_validate_1(self): 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - # self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_running_validate_2(self): """ @@ -71,12 +69,13 @@ def test_locking_running_validate_2(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -129,7 +128,7 @@ def test_locking_running_validate_2(self): 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_running_validate_2_specific_id(self): """ @@ -140,12 +139,13 @@ def test_locking_running_validate_2_specific_id(self): RUNNING backup with pid file AND without active pid is legal, but his status must be changed to ERROR and pid file is deleted """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -227,7 +227,7 @@ def test_locking_running_validate_2_specific_id(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_running_3(self): """ @@ -237,12 +237,13 @@ def test_locking_running_3(self): RUNNING backup without pid file AND without active pid is legal, his status must be changed to ERROR """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -296,22 +297,23 @@ def test_locking_running_3(self): 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_restore_locked(self): """ make node, take full backup, take two page backups, launch validate on PAGE1 and stop it in the middle, launch restore of PAGE2. - Expect restore to fail because validation of - intermediate backup is impossible + Expect restore to sucseed because read-only locks + do not conflict """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -334,24 +336,12 @@ def test_locking_restore_locked(self): node.cleanup() - try: - self.restore_node(backup_dir, 'node', node) - self.assertEqual( - 1, 0, - "Expecting Error because restore without whole chain validation " - "is prohibited unless --no-validate provided.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Cannot lock backup {0} directory\n".format(full_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.restore_node(backup_dir, 'node', node) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() - def test_locking_restore_locked_without_validation(self): + def test_concurrent_delete_and_restore(self): """ make node, take full backup, take page backup, launch validate on FULL and stop it in the middle, @@ -359,12 +349,13 @@ def test_locking_restore_locked_without_validation(self): Expect restore to fail because validation of intermediate backup is impossible """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -376,10 +367,11 @@ def test_locking_restore_locked_without_validation(self): # PAGE1 restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') - gdb = self.validate_pb( + gdb = self.delete_pb( backup_dir, 'node', backup_id=backup_id, gdb=True) - gdb.set_breakpoint('pgBackupValidate') + # gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() node.cleanup() @@ -397,14 +389,14 @@ def test_locking_restore_locked_without_validation(self): self.assertTrue( "Backup {0} is used without validation".format( restore_id) in e.message and - 'is using backup {0} and still is running'.format( + 'is using backup {0}, and is still running'.format( backup_id) in e.message and 'ERROR: Cannot lock backup' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_concurrent_validate_and_backup(self): """ @@ -412,12 +404,13 @@ def test_locking_concurrent_validate_and_backup(self): and stop it in the middle, take page backup. Expect PAGE backup to be successfully executed """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -439,7 +432,7 @@ def test_locking_concurrent_validate_and_backup(self): self.backup_node(backup_dir, 'node', node, backup_type='page') # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_locking_concurren_restore_and_delete(self): """ @@ -447,12 +440,13 @@ def test_locking_concurren_restore_and_delete(self): and stop it in the middle, delete full backup. Expect it to fail. """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -467,7 +461,6 @@ def test_locking_concurren_restore_and_delete(self): gdb.set_breakpoint('create_data_directories') gdb.run_until_break() - # This PAGE backup is expected to be successfull try: self.delete_pb(backup_dir, 'node', full_id) self.assertEqual( @@ -483,17 +476,16 @@ def test_locking_concurren_restore_and_delete(self): repr(e.message), self.cmd)) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb.kill() def test_backup_directory_name(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -537,4 +529,101 @@ def test_backup_directory_name(self): self.show_pb(backup_dir, 'node', page_id_2)) # Clean after yourself - self.del_test_dir(module_name, fname) + + def test_empty_lock_file(self): + """ + https://github.com/postgrespro/pg_probackup/issues/308 + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=100) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + with open(lockfile, "w+") as f: + f.truncate() + + out = self.validate_pb(backup_dir, 'node', backup_id) + + self.assertIn( + "Waiting 30 seconds on empty exclusive lock for backup", out) + +# lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') +# with open(lockfile, "w+") as f: +# f.truncate() +# +# p1 = self.validate_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=validate.log']) +# sleep(3) +# p2 = self.delete_pb(backup_dir, 'node', backup_id, asynchronous=True, +# options=['--log-level-file=LOG', '--log-filename=delete.log']) +# +# p1.wait() +# p2.wait() + + def test_shared_lock(self): + """ + Make sure that shared lock leaves no files with pids + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node) + + lockfile_excl = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') + lockfile_shr = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup_ro.pid') + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + + gdb = self.validate_pb(backup_dir, 'node', backup_id, gdb=True) + + gdb.set_breakpoint('validate_one_page') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + os.path.exists(lockfile_shr), + "File should exist: {0}".format(lockfile_shr)) + + self.validate_pb(backup_dir, 'node', backup_id) + + self.assertFalse( + os.path.exists(lockfile_excl), + "File should not exist: {0}".format(lockfile_excl)) + + self.assertFalse( + os.path.exists(lockfile_shr), + "File should not exist: {0}".format(lockfile_shr)) + diff --git a/tests/logging.py b/tests/logging_test.py similarity index 68% rename from tests/logging.py rename to tests/logging_test.py index efde1d0b9..c5cdfa344 100644 --- a/tests/logging.py +++ b/tests/logging_test.py @@ -3,22 +3,22 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import datetime -module_name = 'logging' - - class LogTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-2154 def test_log_rotation(self): - fname = self.id().split('.')[3] + """ + """ + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -39,17 +39,13 @@ def test_log_rotation(self): gdb.run_until_break() gdb.continue_execution_until_exit() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_log_filename_strftime(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -72,17 +68,13 @@ def test_log_filename_strftime(self): self.assertTrue(os.path.isfile(path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_truncate_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -147,17 +139,13 @@ def test_truncate_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_unlink_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -219,17 +207,13 @@ def test_unlink_rotation_file(self): os.stat(log_file_path).st_size, log_file_size) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_garbage_in_rotation_file(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -255,11 +239,8 @@ def test_garbage_in_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) # mangle .rotation file - with open(rotation_file_path, "wt", 0) as f: + with open(rotation_file_path, "w+b", 0) as f: f.write(b"blah") - f.flush() - f.close - output = self.backup_node( backup_dir, 'node', node, options=[ @@ -298,5 +279,67 @@ def test_garbage_in_rotation_file(self): os.stat(log_file_path).st_size, log_file_size) - # Clean after yourself - self.del_test_dir(module_name, fname) + def test_issue_274(self): + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.restore_node(backup_dir, 'node', replica) + + # Settings for Replica + self.set_replica(node, replica, synchronous=True) + self.set_archiving(backup_dir, 'node', replica, replica=True) + self.set_auto_conf(replica, {'port': replica.port}) + + replica.slow_start(replica=True) + + node.safe_psql( + "postgres", + "create table t_heap as select i as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,45600) i") + + log_dir = os.path.join(backup_dir, "somedir") + + try: + self.backup_node( + backup_dir, 'node', replica, backup_type='page', + options=[ + '--log-level-console=verbose', '--log-level-file=verbose', + '--log-directory={0}'.format(log_dir), '-j1', + '--log-filename=somelog.txt', '--archive-timeout=5s', + '--no-validate', '--log-rotation-size=100KB']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because of archiving timeout" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: WAL segment', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + log_file_path = os.path.join( + log_dir, 'somelog.txt') + + self.assertTrue(os.path.isfile(log_file_path)) + + with open(log_file_path, "r+") as f: + log_content = f.read() + + self.assertIn('INFO: command:', log_content) diff --git a/tests/merge.py b/tests/merge_test.py similarity index 80% rename from tests/merge.py rename to tests/merge_test.py index 3444056d2..1d40af7f7 100644 --- a/tests/merge.py +++ b/tests/merge_test.py @@ -7,9 +7,7 @@ import shutil from datetime import datetime, timedelta import time - -module_name = "merge" - +import subprocess class MergeTest(ProbackupTest, unittest.TestCase): @@ -17,12 +15,11 @@ def test_basic_merge_full_page(self): """ Test MERGE command, it merges FULL backup with target PAGE backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=["--data-checksums"]) self.init_pb(backup_dir) @@ -99,19 +96,15 @@ def test_basic_merge_full_page(self): count2 = node.execute("postgres", "select count(*) from test") self.assertEqual(count1, count2) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - def test_merge_compressed_backups(self): """ Test MERGE command with compressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=["--data-checksums"]) self.init_pb(backup_dir) @@ -162,20 +155,17 @@ def test_merge_compressed_backups(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_backups_1(self): """ Test MERGE command with compressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, initdb_params=["--data-checksums"]) self.init_pb(backup_dir) self.add_instance(backup_dir, "node", node) @@ -234,22 +224,17 @@ def test_merge_compressed_backups_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -309,22 +294,17 @@ def test_merge_compressed_and_uncompressed_backups(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups_1(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -386,22 +366,17 @@ def test_merge_compressed_and_uncompressed_backups_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) def test_merge_compressed_and_uncompressed_backups_2(self): """ Test MERGE command with compressed and uncompressed backups """ - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, "backup") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=["--data-checksums"], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -459,11 +434,6 @@ def test_merge_compressed_and_uncompressed_backups_2(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - node.cleanup() - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") def test_merge_tablespaces(self): """ @@ -472,14 +442,10 @@ def test_merge_tablespaces(self): tablespace, take page backup, merge it and restore """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -550,14 +516,10 @@ def test_merge_tablespaces_1(self): drop first tablespace and take delta backup, merge it and restore """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off' - } ) self.init_pb(backup_dir) @@ -622,9 +584,6 @@ def test_merge_tablespaces_1(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_page_truncate(self): """ make node, create table, take full backup, @@ -632,18 +591,16 @@ def test_merge_page_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -702,19 +659,11 @@ def test_merge_page_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_delta_truncate(self): """ make node, create table, take full backup, @@ -722,18 +671,16 @@ def test_merge_delta_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -792,19 +739,11 @@ def test_merge_delta_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_ptrack_truncate(self): """ make node, create table, take full backup, @@ -813,26 +752,23 @@ def test_merge_ptrack_truncate(self): restore last page backup and check data correctness """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - ptrack_enable=True, - pg_options={'autovacuum': 'off'}) + ptrack_enable=True) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -869,7 +805,7 @@ def test_merge_ptrack_truncate(self): self.validate_pb(backup_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -890,19 +826,11 @@ def test_merge_ptrack_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_merge_delta_delete(self): """ @@ -910,14 +838,12 @@ def test_merge_delta_delete(self): alter tablespace location, take delta backup, merge full and delta, restore database. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -963,7 +889,7 @@ def test_merge_delta_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -987,19 +913,17 @@ def test_merge_delta_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_continue_failed_merge(self): """ Check that failed MERGE can be continued """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join( - module_name, fname, 'node'), + self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1063,18 +987,16 @@ def test_continue_failed_merge(self): node.cleanup() self.restore_node(backup_dir, 'node', node) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1093,7 +1015,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): old_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP self.backup_node( @@ -1109,7 +1031,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): new_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP backup_id_2 = self.backup_node( @@ -1161,17 +1083,15 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_continue_failed_merge_2(self): """ Check that failed MERGE on delete can be continued """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1231,18 +1151,17 @@ def test_continue_failed_merge_2(self): # Try to continue failed MERGE self.merge_backup(backup_dir, "node", backup_id) - # Clean after yourself - self.del_test_dir(module_name, fname) def test_continue_failed_merge_3(self): """ Check that failed MERGE cannot be continued if intermediate backup is missing. """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -1327,17 +1246,13 @@ def test_continue_failed_merge_3(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_merge_different_compression_algo(self): """ Check that backups with different compression algorithms can be merged """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1380,17 +1295,14 @@ def test_merge_different_compression_algo(self): self.merge_backup(backup_dir, "node", backup_id) - self.del_test_dir(module_name, fname) - def test_merge_different_wal_modes(self): """ Check that backups with different wal modes can be merged correctly """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1422,20 +1334,18 @@ def test_merge_different_wal_modes(self): self.assertEqual( 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) - self.del_test_dir(module_name, fname) - def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control for writing will not result in losing backup metadata """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1473,22 +1383,20 @@ def test_crash_after_opening_backup_control_1(self): self.assertEqual( 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) - self.del_test_dir(module_name, fname) - - @unittest.skip("skip") + # @unittest.skip("skip") def test_crash_after_opening_backup_control_2(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1508,7 +1416,7 @@ def test_crash_after_opening_backup_control_2(self): path = node.safe_psql( 'postgres', - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() fsm_path = path + '_fsm' @@ -1529,8 +1437,8 @@ def test_crash_after_opening_backup_control_2(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('sprintf') - gdb.continue_execution_until_break(1) +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(1) gdb._execute('signal SIGKILL') @@ -1565,22 +1473,20 @@ def test_crash_after_opening_backup_control_2(self): self.compare_pgdata(pgdata, pgdata_restored) - self.del_test_dir(module_name, fname) - - @unittest.skip("skip") + # @unittest.skip("skip") def test_losing_file_after_failed_merge(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1601,7 +1507,7 @@ def test_losing_file_after_failed_merge(self): path = node.safe_psql( 'postgres', - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( 'postgres', @@ -1622,8 +1528,8 @@ def test_losing_file_after_failed_merge(self): gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() - gdb.set_breakpoint('sprintf') - gdb.continue_execution_until_break(20) +# gdb.set_breakpoint('sprintf') +# gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1657,18 +1563,16 @@ def test_losing_file_after_failed_merge(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1682,7 +1586,7 @@ def test_failed_merge_after_delete(self): dboid = node.safe_psql( "postgres", - "select oid from pg_database where datname = 'testdb'").rstrip() + "select oid from pg_database where datname = 'testdb'").decode('utf-8').rstrip() # take FULL backup full_id = self.backup_node( @@ -1739,18 +1643,16 @@ def test_failed_merge_after_delete(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_1(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1816,18 +1718,16 @@ def test_failed_merge_after_delete_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_2(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1879,18 +1779,16 @@ def test_failed_merge_after_delete_2(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - def test_failed_merge_after_delete_3(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1968,22 +1866,22 @@ def test_failed_merge_after_delete_3(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_merge_backup_from_future(self): """ take FULL backup, table PAGE backup from future, try to merge page with FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2018,12 +1916,10 @@ def test_merge_backup_from_future(self): backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2046,17 +1942,12 @@ def test_merge_backup_from_future(self): {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - 'postgres', - 'SELECT * from pgbench_accounts') + result_new = node_restored.table_checksum("pgbench_accounts") - self.assertTrue(result, result_new) + self.assertEqual(result, result_new) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_merge_multiple_descendants(self): """ @@ -2069,12 +1960,11 @@ def test_merge_multiple_descendants(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2243,9 +2133,6 @@ def test_merge_multiple_descendants(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_smart_merge(self): """ @@ -2255,13 +2142,12 @@ def test_smart_merge(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2303,19 +2189,16 @@ def test_smart_merge(self): with open(logfile, 'r') as f: logfile_content = f.read() - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - def test_idempotent_merge(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2379,21 +2262,17 @@ def test_idempotent_merge(self): self.assertEqual( page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) - self.del_test_dir(module_name, fname, [node]) - def test_merge_correct_inheritance(self): """ Make sure that backup metainformation fields 'note' and 'expire-time' are correctly inherited during merge """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2434,21 +2313,17 @@ def test_merge_correct_inheritance(self): page_meta['expire-time'], self.show_pb(backup_dir, 'node', page_id)['expire-time']) - self.del_test_dir(module_name, fname, [node]) - def test_merge_correct_inheritance_1(self): """ Make sure that backup metainformation fields 'note' and 'expire-time' are correctly inherited during merge """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2484,8 +2359,6 @@ def test_merge_correct_inheritance_1(self): 'expire-time', self.show_pb(backup_dir, 'node', page_id)) - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multi_timeline_merge(self): @@ -2500,13 +2373,11 @@ def test_multi_timeline_merge(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2568,11 +2439,10 @@ def test_multi_timeline_merge(self): self.merge_backup(backup_dir, 'node', page_id) - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -2581,8 +2451,7 @@ def test_multi_timeline_merge(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = node_restored.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) @@ -2602,9 +2471,6 @@ def test_multi_timeline_merge(self): '--amcheck', '-d', 'postgres', '-p', str(node_restored.port)]) - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) - # @unittest.skip("skip") # @unittest.expectedFailure def test_merge_page_header_map_retry(self): @@ -2612,13 +2478,13 @@ def test_merge_page_header_map_retry(self): page header map cannot be trusted when running retry """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2652,8 +2518,264 @@ def test_merge_page_header_map_retry(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_missing_data_file(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Add data + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # Change data + pgbench = node.pgbench(options=['-T', '5', '-c', '1']) + pgbench.wait() + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + path = node.safe_psql( + 'postgres', + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', path) + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), + logfile_content) + + # @unittest.skip("skip") + def test_missing_non_data_file(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + gdb = self.merge_backup( + backup_dir, "node", delta_id, + options=['--log-level-file=VERBOSE'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + # remove data file in incremental backup + file_to_remove = os.path.join( + backup_dir, 'backups', + 'node', delta_id, 'database', 'backup_label') + + os.remove(file_to_remove) + + gdb.continue_execution_until_error() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertIn( + 'ERROR: File "{0}" is not found'.format(file_to_remove), + logfile_content) + + self.assertIn( + 'ERROR: Backup files merging failed', + logfile_content) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + + self.assertEqual( + 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + + # @unittest.skip("skip") + def test_merge_remote_mode(self): + """ + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + # DELTA backup + delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + + self.set_config(backup_dir, 'node', options=['--retention-window=1']) + + backups = os.path.join(backup_dir, 'backups', 'node') + with open( + os.path.join( + backups, full_id, "backup.control"), "a") as conf: + conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=5))) + + gdb = self.backup_node( + backup_dir, "node", node, + options=['--log-level-file=VERBOSE', '--merge-expired'], gdb=True) + gdb.set_breakpoint('merge_files') + gdb.run_until_break() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + + with open(logfile, "w+") as f: + f.truncate() + + gdb.continue_execution_until_exit() + + logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(logfile, 'r') as f: + logfile_content = f.read() + + self.assertNotIn( + 'SSH', logfile_content) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + + def test_merge_pg_filenode_map(self): + """ + https://github.com/postgrespro/pg_probackup/issues/320 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + initdb_params=['--data-checksums']) + node1.cleanup() + + node.pgbench_init(scale=5) + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1']) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + node.safe_psql( + 'postgres', + 'reindex index pg_type_oid_index') + + backup_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + + self.merge_backup(backup_dir, 'node', backup_id) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') + + def test_unfinished_merge(self): + """ Test when parent has unfinished merge with a different backup. """ + self._check_gdb_flag_or_skip_test() + cases = [('fail_merged', 'write_backup_filelist', ['MERGED', 'MERGING', 'OK']), + ('fail_merging', 'pgBackupWriteControl', ['MERGING', 'OK', 'OK'])] + + for name, terminate_at, states in cases: + node_name = 'node_' + name + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, name) + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, node_name), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, node_name, node) + self.set_archiving(backup_dir, node_name, node) + node.slow_start() + + full_id=self.backup_node(backup_dir, node_name, node, options=['--stream']) + + backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') + second_backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') + + gdb = self.merge_backup(backup_dir, node_name, backup_id, gdb=True) + gdb.set_breakpoint(terminate_at) + gdb.run_until_break() + + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + print(self.show_pb(backup_dir, node_name, as_json=False, as_text=True)) + + backup_infos = self.show_pb(backup_dir, node_name) + self.assertEqual(len(backup_infos), len(states)) + for expected, real in zip(states, backup_infos): + self.assertEqual(expected, real['status']) + + with self.assertRaisesRegex(ProbackupException, + f"Full backup {full_id} has unfinished merge with backup {backup_id}"): + self.merge_backup(backup_dir, node_name, second_backup_id, gdb=False) # 1. Need new test with corrupted FULL backup # 2. different compression levels diff --git a/tests/option.py b/tests/option_test.py similarity index 72% rename from tests/option.py rename to tests/option_test.py index 023a0c2c6..d1e8cb3a6 100644 --- a/tests/option.py +++ b/tests/option_test.py @@ -1,10 +1,7 @@ import unittest import os from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -module_name = 'option' - +import locale class OptionTest(ProbackupTest, unittest.TestCase): @@ -18,15 +15,6 @@ def test_help_1(self): help_out.read().decode("utf-8") ) - # @unittest.skip("skip") - def test_version_2(self): - """help options""" - with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: - self.assertIn( - version_out.read().decode("utf-8"), - self.run_pb(["--version"]) - ) - # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" @@ -36,18 +24,18 @@ def test_without_backup_path_3(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: BACKUP_PATH (-B, --backup-path)', + 'ERROR: No backup catalog path specified.\n' + \ + 'Please specify it either using environment variable BACKUP_PATH or\n' + \ + 'command line option --backup-path (-B)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # @unittest.skip("skip") def test_options_4(self): """check options test""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -59,7 +47,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: --instance', + 'ERROR: Required parameter not specified: --instance', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -70,7 +58,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: BACKUP_MODE (-b, --backup-mode)', + 'ERROR: No backup mode specified.\nPlease specify it either using environment variable BACKUP_MODE or\ncommand line option --backup-mode (-b)', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -81,7 +69,7 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: invalid backup-mode "bad"', + 'ERROR: Invalid backup-mode "bad"', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) @@ -107,29 +95,20 @@ def test_options_4(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - "option requires an argument -- 'i'", + "Option '-i' requires an argument", e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_options_5(self): """check options test""" - fname = self.id().split(".")[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) output = self.init_pb(backup_dir) - self.assertIn( - "INFO: Backup catalog", - output) + self.assertIn(f"INFO: Backup catalog '{backup_dir}' successfully initialized", output) - self.assertIn( - "successfully inited", - output) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -224,5 +203,51 @@ def test_options_5(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_help_6(self): + """help options""" + if ProbackupTest.enable_nls: + if check_locale('ru_RU.utf-8'): + self.test_env['LC_ALL'] = 'ru_RU.utf-8' + with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + self.assertEqual( + self.run_pb(["--help"]), + help_out.read().decode("utf-8") + ) + else: + self.skipTest( + "Locale ru_RU.utf-8 doesn't work. You need install ru_RU.utf-8 locale for this test") + else: + self.skipTest( + 'You need configure PostgreSQL with --enabled-nls option for this test') + + # @unittest.skip("skip") + def test_options_no_scale_units(self): + """check --no-scale-units option""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + # check that --no-scale-units option works correctly + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node"]) + self.assertIn(container=output, member="archive-timeout = 5min") + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--no-scale-units"]) + self.assertIn(container=output, member="archive-timeout = 300") + self.assertNotIn(container=output, member="archive-timeout = 300s") + # check that we have now quotes ("") in json output + output = self.run_pb(["show-config", "--backup-path", backup_dir, "--instance=node", "--no-scale-units", "--format=json"]) + self.assertIn(container=output, member='"archive-timeout": 300,') + self.assertIn(container=output, member='"retention-redundancy": 0,') + self.assertNotIn(container=output, member='"archive-timeout": "300",') + +def check_locale(locale_name): + ret=True + old_locale = locale.setlocale(locale.LC_CTYPE,"") + try: + locale.setlocale(locale.LC_CTYPE, locale_name) + except locale.Error: + ret=False + finally: + locale.setlocale(locale.LC_CTYPE, old_locale) + return ret diff --git a/tests/page.py b/tests/page_test.py similarity index 85% rename from tests/page.py rename to tests/page_test.py index 201f825e8..a66d6d413 100644 --- a/tests/page.py +++ b/tests/page_test.py @@ -6,9 +6,7 @@ import subprocess import gzip import shutil - -module_name = 'page' - +import time class PageTest(ProbackupTest, unittest.TestCase): @@ -20,18 +18,16 @@ def test_basic_page_vacuum_truncate(self): take page backup, take second page backup, restore last page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '300s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '300s'}) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -89,19 +85,11 @@ def test_basic_page_vacuum_truncate(self): node_restored.slow_start() # Logical comparison - result1 = node.safe_psql( - "postgres", - "select * from t_heap") - - result2 = node_restored.safe_psql( - "postgres", - "select * from t_heap") + result1 = node.table_checksum("t_heap") + result2 = node_restored.table_checksum("t_heap") self.assertEqual(result1, result2) - # Clean after yourself - self.del_test_dir(module_name, fname, [node, node_restored]) - # @unittest.skip("skip") def test_page_vacuum_truncate_1(self): """ @@ -110,13 +98,11 @@ def test_page_vacuum_truncate_1(self): take page backup, insert some data, take second page backup and check data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -161,7 +147,7 @@ def test_page_vacuum_truncate_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -173,9 +159,6 @@ def test_page_vacuum_truncate_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_stream(self): """ @@ -183,10 +166,9 @@ def test_page_stream(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -205,7 +187,7 @@ def test_page_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full', options=['--stream']) @@ -216,7 +198,7 @@ def test_page_stream(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=['--stream', '-j', '4']) @@ -237,7 +219,7 @@ def test_page_stream(self): ' CMD: {1}'.format(repr(self.output), self.cmd)) node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -256,13 +238,10 @@ def test_page_stream(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_archive(self): """ @@ -270,10 +249,9 @@ def test_page_archive(self): restore them and check data correctness """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -290,7 +268,7 @@ def test_page_archive(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - full_result = node.execute("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='full') @@ -300,7 +278,7 @@ def test_page_archive(self): "insert into t_heap select i as id, " "md5(i::text) as text, md5(i::text)::tsvector as tsvector " "from generate_series(100, 200) i") - page_result = node.execute("postgres", "SELECT * FROM t_heap") + page_result = node.table_checksum("t_heap") page_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page', options=["-j", "4"]) @@ -326,7 +304,7 @@ def test_page_archive(self): node.slow_start() - full_result_new = node.execute("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -350,30 +328,25 @@ def test_page_archive(self): node.slow_start() - page_result_new = node.execute("postgres", "SELECT * FROM t_heap") + page_result_new = node.table_checksum("t_heap") self.assertEqual(page_result, page_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_multiple_segments(self): """ Make node, create table with multiple segments, write some data to it, check page and data correctness """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', - 'autovacuum': 'off', 'full_page_writes': 'off'}) self.init_pb(backup_dir) @@ -393,7 +366,7 @@ def test_page_multiple_segments(self): pgbench.wait() # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") # PAGE BACKUP self.backup_node(backup_dir, 'node', node, backup_type='page') @@ -402,7 +375,7 @@ def test_page_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -421,8 +394,7 @@ def test_page_multiple_segments(self): self.set_auto_conf(restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') @@ -430,9 +402,6 @@ def test_page_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_delete(self): """ @@ -440,14 +409,12 @@ def test_page_delete(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -481,7 +448,7 @@ def test_page_delete(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -503,9 +470,6 @@ def test_page_delete(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_delete_1(self): """ @@ -513,15 +477,13 @@ def test_page_delete_1(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'autovacuum': 'off' } ) @@ -559,7 +521,7 @@ def test_page_delete_1(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored') + base_dir=os.path.join(self.module_name, self.fname, 'node_restored') ) node_restored.cleanup() @@ -582,26 +544,22 @@ def test_page_delete_1(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_parallel_pagemap(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={ "hot_standby": "on" } ) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), ) self.init_pb(backup_dir) @@ -657,18 +615,16 @@ def test_parallel_pagemap(self): # Clean after yourself node.cleanup() node_restored.cleanup() - self.del_test_dir(module_name, fname) def test_parallel_pagemap_1(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') # Initialize instance and backup directory node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={} ) @@ -709,7 +665,6 @@ def test_parallel_pagemap_1(self): # Clean after yourself node.cleanup() - self.del_test_dir(module_name, fname) # @unittest.skip("skip") def test_page_backup_with_lost_wal_segment(self): @@ -720,12 +675,11 @@ def test_page_backup_with_lost_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -757,8 +711,6 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -782,8 +734,6 @@ def test_page_backup_with_lost_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'is absent' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -794,9 +744,6 @@ def test_page_backup_with_lost_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_backup_with_corrupted_wal_segment(self): """ @@ -806,12 +753,11 @@ def test_page_backup_with_corrupted_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -872,8 +818,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -896,8 +840,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( file) in e.message, @@ -909,9 +851,6 @@ def test_page_backup_with_corrupted_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_backup_with_alien_wal_segment(self): """ @@ -923,18 +862,17 @@ def test_page_backup_with_alien_wal_segment(self): expecting error because of alien wal segment make sure that backup status is 'ERROR' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) alien_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'alien_node'), + base_dir=os.path.join(self.module_name, self.fname, 'alien_node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -956,7 +894,7 @@ def test_page_backup_with_alien_wal_segment(self): "create table t_heap as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,1000) i;") + "from generate_series(0,10000) i;") alien_node.safe_psql( "postgres", @@ -968,7 +906,7 @@ def test_page_backup_with_alien_wal_segment(self): "create table t_heap_alien as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,100000) i;") + "from generate_series(0,10000) i;") # copy latest wal segment wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') @@ -979,9 +917,9 @@ def test_page_backup_with_alien_wal_segment(self): file = os.path.join(wals_dir, filename) file_destination = os.path.join( os.path.join(backup_dir, 'wal', 'node'), filename) -# file = os.path.join(wals_dir, '000000010000000000000004') - print(file) - print(file_destination) + start = time.time() + while not os.path.exists(file_destination) and time.time() - start < 20: + time.sleep(0.1) os.remove(file_destination) os.rename(file, file_destination) @@ -997,8 +935,6 @@ def test_page_backup_with_alien_wal_segment(self): self.output, self.cmd)) except ProbackupException as e: self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'to be archived' in e.message and 'Could not read WAL record at' in e.message and 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( @@ -1020,8 +956,6 @@ def test_page_backup_with_alien_wal_segment(self): "Output: {0} \n CMD: {1}".format( self.output, self.cmd)) except ProbackupException as e: - self.assertIn('INFO: Wait for WAL segment', e.message) - self.assertIn('to be archived', e.message) self.assertIn('Could not read WAL record at', e.message) self.assertIn('WAL file is from different database system: ' 'WAL file database system identifier is', e.message) @@ -1034,20 +968,16 @@ def test_page_backup_with_alien_wal_segment(self): self.show_pb(backup_dir, 'node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_multithread_page_backup_with_toast(self): """ make node, create toast, do multithread PAGE backup """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1067,9 +997,6 @@ def test_multithread_page_backup_with_toast(self): backup_dir, 'node', node, backup_type='page', options=["-j", "4"]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_page_create_db(self): """ @@ -1077,16 +1004,14 @@ def test_page_create_db(self): restore database and check it presense """ self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', - 'autovacuum': 'off' } ) @@ -1119,7 +1044,7 @@ def test_page_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1180,9 +1105,6 @@ def test_page_create_db(self): repr(e.message), self.cmd) ) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multi_timeline_page(self): @@ -1197,13 +1119,11 @@ def test_multi_timeline_page(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1266,11 +1186,10 @@ def test_multi_timeline_page(self): pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql( - "postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1279,8 +1198,7 @@ def test_multi_timeline_page(self): self.set_auto_conf(node_restored, {'port': node_restored.port}) node_restored.slow_start() - result_new = node_restored.safe_psql( - "postgres", "select * from pgbench_accounts") + result_new = node_restored.table_checksum("pgbench_accounts") self.assertEqual(result, result_new) @@ -1322,9 +1240,6 @@ def test_multi_timeline_page(self): backup_list[4]['id']) self.assertEqual(backup_list[5]['current-tli'], 7) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_multitimeline_page_1(self): @@ -1336,13 +1251,12 @@ def test_multitimeline_page_1(self): P must have F as parent """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off', 'wal_log_hints': 'on'}) + pg_options={'wal_log_hints': 'on'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1392,7 +1306,7 @@ def test_multitimeline_page_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -1403,23 +1317,18 @@ def test_multitimeline_page_1(self): self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_page_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1498,7 +1407,7 @@ def test_page_pg_resetxlog(self): # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( @@ -1507,5 +1416,49 @@ def test_page_pg_resetxlog(self): # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + def test_page_huge_xlog_record(self): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'max_locks_per_transaction': '1000', + 'work_mem': '100MB', + 'temp_buffers': '100MB', + 'wal_buffers': '128MB', + 'wal_level' : 'logical', + }) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Do full backup + self.backup_node(backup_dir, 'node', node, backup_type='full') + show_backup = self.show_pb(backup_dir,'node')[0] + + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Originally client had the problem at the transaction that (supposedly) + # deletes a lot of temporary tables (probably it was client disconnect). + # It generated ~40MB COMMIT WAL record. + # + # `pg_logical_emit_message` is much simpler and faster way to generate + # such huge record. + node.safe_psql( + "postgres", + "select pg_logical_emit_message(False, 'z', repeat('o', 60*1000*1000))") + + # Do page backup + self.backup_node(backup_dir, 'node', node, backup_type='page') + + show_backup = self.show_pb(backup_dir,'node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") diff --git a/tests/pgpro2068.py b/tests/pgpro2068_test.py similarity index 61% rename from tests/pgpro2068.py rename to tests/pgpro2068_test.py index 253be3441..04f0eb6fa 100644 --- a/tests/pgpro2068.py +++ b/tests/pgpro2068_test.py @@ -9,18 +9,16 @@ from testgres import ProcessType -module_name = '2068' - - class BugTest(ProbackupTest, unittest.TestCase): def test_minrecpoint_on_replica(self): """ https://jira.postgrespro.ru/browse/PGPRO-2068 """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -31,7 +29,7 @@ def test_minrecpoint_on_replica(self): 'bgwriter_lru_multiplier': '4.0', 'max_wal_size': '256MB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -43,7 +41,7 @@ def test_minrecpoint_on_replica(self): # start replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica, options=['-R']) @@ -55,11 +53,6 @@ def test_minrecpoint_on_replica(self): replica, {'port': replica.port, 'restart_after_crash': 'off'}) - # we need those later - node.safe_psql( - "postgres", - "CREATE EXTENSION plpythonu") - node.safe_psql( "postgres", "CREATE EXTENSION pageinspect") @@ -87,7 +80,6 @@ def test_minrecpoint_on_replica(self): # get pids of replica background workers startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] - bgwriter_pid = replica.auxiliary_pids[ProcessType.BackgroundWriter][0] # break checkpointer on UpdateLastRemovedPtr gdb_checkpointer = self.gdb_attach(checkpointer_pid) @@ -110,7 +102,7 @@ def test_minrecpoint_on_replica(self): pgbench.stdout.close() # kill someone, we need a crash - os.kill(int(bgwriter_pid), 9) + replica.kill(someone=ProcessType.BackgroundWriter) gdb_recovery._execute('detach') gdb_checkpointer._execute('detach') @@ -134,48 +126,37 @@ def test_minrecpoint_on_replica(self): recovery_config, "recovery_target_action = 'pause'") replica.slow_start(replica=True) + current_xlog_lsn_query = 'SELECT pg_last_wal_replay_lsn() INTO current_xlog_lsn' if self.get_version(node) < 100000: - script = ''' -DO -$$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("SELECT min_recovery_end_lsn as lsn FROM pg_control_recovery()")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpythonu; -''' - else: - script = ''' + current_xlog_lsn_query = 'SELECT min_recovery_end_location INTO current_xlog_lsn FROM pg_control_recovery()' + + script = f''' DO $$ -relations = plpy.execute("select class.oid from pg_class class WHERE class.relkind IN ('r', 'i', 't', 'm') and class.relpersistence = 'p'") -current_xlog_lsn = plpy.execute("select pg_last_wal_replay_lsn() as lsn")[0]['lsn'] -plpy.notice('CURRENT LSN: {0}'.format(current_xlog_lsn)) -found_corruption = False -for relation in relations: - pages_from_future = plpy.execute("with number_of_blocks as (select blknum from generate_series(0, pg_relation_size({0}) / 8192 -1) as blknum) select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid from number_of_blocks, page_header(get_raw_page('{0}'::oid::regclass::text, number_of_blocks.blknum::int)) where lsn > '{1}'::pg_lsn".format(relation['oid'], current_xlog_lsn)) - - if pages_from_future.nrows() == 0: - continue - - for page in pages_from_future: - plpy.notice('Found page from future. OID: {0}, BLKNUM: {1}, LSN: {2}'.format(relation['oid'], page['blknum'], page['lsn'])) - found_corruption = True -if found_corruption: - plpy.error('Found Corruption') -$$ LANGUAGE plpythonu; -''' +DECLARE + roid oid; + current_xlog_lsn pg_lsn; + pages_from_future RECORD; + found_corruption bool := false; +BEGIN + {current_xlog_lsn_query}; + RAISE NOTICE 'CURRENT LSN: %', current_xlog_lsn; + FOR roid IN select oid from pg_class class where relkind IN ('r', 'i', 't', 'm') and relpersistence = 'p' LOOP + FOR pages_from_future IN + with number_of_blocks as (select blknum from generate_series(0, pg_relation_size(roid) / 8192 -1) as blknum ) + select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid + from number_of_blocks, page_header(get_raw_page(roid::regclass::text, number_of_blocks.blknum::int)) + where lsn > current_xlog_lsn LOOP + RAISE NOTICE 'Found page from future. OID: %, BLKNUM: %, LSN: %', roid, pages_from_future.blknum, pages_from_future.lsn; + found_corruption := true; + END LOOP; + END LOOP; + IF found_corruption THEN + RAISE 'Found Corruption'; + END IF; +END; +$$ LANGUAGE plpgsql; +'''.format(current_xlog_lsn_query=current_xlog_lsn_query) # Find blocks from future replica.safe_psql( @@ -188,6 +169,3 @@ def test_minrecpoint_on_replica(self): # do basebackup # do pg_probackup, expect error - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/pgpro560.py b/tests/pgpro560_test.py similarity index 83% rename from tests/pgpro560.py rename to tests/pgpro560_test.py index 53c7914a2..b665fd200 100644 --- a/tests/pgpro560.py +++ b/tests/pgpro560_test.py @@ -6,9 +6,6 @@ from time import sleep -module_name = 'pgpro560' - - class CheckSystemID(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -20,27 +17,27 @@ def test_pgpro560_control_file_loss(self): make backup check that backup failed """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') - os.remove(file) + # Not delete this file permanently + os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) try: self.backup_node(backup_dir, 'node', node, options=['--stream']) # we should die here because exception is what we expect to happen self.assertEqual( - 1, 0, - "Expecting Error because pg_control was deleted.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) + 1, 0, + "Expecting Error because pg_control was deleted.\n " + "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( 'ERROR: Could not open file' in e.message and @@ -48,8 +45,8 @@ def test_pgpro560_control_file_loss(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) + # Return this file to avoid Postger fail + os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) def test_pgpro560_systemid_mismatch(self): """ @@ -58,21 +55,20 @@ def test_pgpro560_systemid_mismatch(self): feed to backup PGDATA from node1 and PGPORT from node2 check that backup failed """ - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) node1.slow_start() node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2'), + base_dir=os.path.join(self.module_name, self.fname, 'node2'), set_replication=True, initdb_params=['--data-checksums']) node2.slow_start() - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) @@ -125,6 +121,3 @@ def test_pgpro560_systemid_mismatch(self): e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/pgpro589.py b/tests/pgpro589_test.py similarity index 90% rename from tests/pgpro589.py rename to tests/pgpro589_test.py index d6381a8b5..8ce8e1f56 100644 --- a/tests/pgpro589.py +++ b/tests/pgpro589_test.py @@ -5,9 +5,6 @@ import subprocess -module_name = 'pgpro589' - - class ArchiveCheck(ProbackupTest, unittest.TestCase): def test_pgpro589(self): @@ -17,12 +14,11 @@ def test_pgpro589(self): check that backup status equal to ERROR check that no files where copied to backup catalogue """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -74,6 +70,3 @@ def test_pgpro589(self): "\n Start LSN was not found in archive but datafiles where " "copied to backup catalogue.\n For example: {0}\n " "It is not optimal".format(file)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/ptrack.py b/tests/ptrack_test.py similarity index 76% rename from tests/ptrack.py rename to tests/ptrack_test.py index a709afb74..7b5bc416b 100644 --- a/tests/ptrack.py +++ b/tests/ptrack_test.py @@ -10,32 +10,413 @@ from threading import Thread -module_name = 'ptrack' +class PtrackTest(ProbackupTest, unittest.TestCase): + def setUp(self): + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + self.fname = self.id().split('.')[3] + # @unittest.skip("skip") + def test_drop_rel_during_backup_ptrack(self): + """ + drop relation during ptrack backup + """ + self._check_gdb_flag_or_skip_test() -class PtrackTest(ProbackupTest, unittest.TestCase): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql( + "postgres", + "create table t_heap as select i" + " as id from generate_series(0,100) i") + + relative_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + absolute_path = os.path.join(node.data_dir, relative_path) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + # PTRACK backup + gdb = self.backup_node( + backup_dir, 'node', node, backup_type='ptrack', + gdb=True, options=['--log-level-file=LOG']) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + + # REMOVE file + os.remove(absolute_path) + + # File removed, we can proceed with backup + gdb.continue_execution_until_exit() + + pgdata = self.pgdata_content(node.data_dir) + + with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: + log_content = f.read() + self.assertTrue( + 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, + 'File "{0}" should be deleted but it`s not'.format(absolute_path)) + + node.cleanup() + self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + + # Physical comparison + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_ptrack_without_full(self): + """ptrack backup without validated full backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + try: + self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because page backup should not be possible " + "without valid full backup.\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and + "ERROR: Create new full backup before an incremental one" in e.message, + "\n Unexpected Error Message: {0}\n CMD: {1}".format( + repr(e.message), self.cmd)) + + self.assertEqual( + self.show_pb(backup_dir, 'node')[0]['status'], + "ERROR") + + # @unittest.skip("skip") + def test_ptrack_threads(self): + """ptrack multi thread backup mode""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + ptrack_enable=True) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.backup_node( + backup_dir, 'node', node, + backup_type="full", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + self.backup_node( + backup_dir, 'node', node, + backup_type="ptrack", options=["-j", "4"]) + self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + + # @unittest.skip("skip") + def test_ptrack_stop_pg(self): + """ + create node, take full backup, + restart node, check that ptrack backup + can be taken + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + + node.stop() + node.slow_start() + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['--stream']) + + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup(self): + """ + t2 /------P2 + t1 ------F---*-----P1 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) + sleep(15) + + xid = node.safe_psql( + 'postgres', + 'SELECT txid_current()').decode('utf-8').rstrip() + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + node.cleanup() + + # Restore from full backup to create Timeline 2 + print(self.restore_node( + backup_dir, 'node', node, + options=[ + '--recovery-target-xid={0}'.format(xid), + '--recovery-target-action=promote'])) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # @unittest.skip("skip") + def test_ptrack_multi_timeline_backup_1(self): + """ + t2 /------ + t1 ---F---P1---* + + # delete P1 + t2 /------P2 + t1 ---F--------* + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=5) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + node.slow_start() + + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # delete old PTRACK backup + self.delete_pb(backup_dir, 'node', backup_id=ptrack_id) + + # take new PTRACK backup + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + + pgdata = self.pgdata_content(node.data_dir) + + node.cleanup() + + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + balance = node.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').\ + decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # @unittest.skip("skip") + def test_ptrack_eat_my_data(self): + """ + PGPRO-4051 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.pgbench_init(scale=50) + + self.backup_node(backup_dir, 'node', node) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) + + for i in range(10): + print("Iteration: {0}".format(i)) + + sleep(2) + + self.backup_node(backup_dir, 'node', node, backup_type='ptrack') +# pgdata = self.pgdata_content(node.data_dir) +# +# node_restored.cleanup() +# +# self.restore_node(backup_dir, 'node', node_restored) +# pgdata_restored = self.pgdata_content(node_restored.data_dir) +# +# self.compare_pgdata(pgdata, pgdata_restored) + + pgbench.terminate() + pgbench.wait() + + self.switch_wal_segment(node) + + result = node.table_checksum("pgbench_accounts") + + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + self.set_auto_conf( + node_restored, {'port': node_restored.port}) + + node_restored.slow_start() + + balance = node_restored.safe_psql( + 'postgres', + 'select (select sum(tbalance) from pgbench_tellers) - ' + '( select sum(bbalance) from pgbench_branches) + ' + '( select sum(abalance) from pgbench_accounts ) - ' + '(select sum(delta) from pgbench_history) as must_be_zero').decode('utf-8').rstrip() + + self.assertEqual('0', balance) + + # Logical comparison + self.assertEqual( + result, + node.table_checksum("pgbench_accounts"), + 'Data loss') # @unittest.skip("skip") def test_ptrack_simple(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -59,10 +440,10 @@ def test_ptrack_simple(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -82,18 +463,14 @@ def test_ptrack_simple(self): # Logical comparison self.assertEqual( result, - node_restored.safe_psql("postgres", "SELECT * FROM t_heap")) - - # Clean after yourself - self.del_test_dir(module_name, fname) + node_restored.table_checksum("t_heap")) # @unittest.skip("skip") def test_ptrack_unprivileged(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -129,16 +506,17 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -161,10 +539,12 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -175,8 +555,8 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -197,8 +577,10 @@ def test_ptrack_unprivileged(self): "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -209,33 +591,52 @@ def test_ptrack_unprivileged(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - - if node.major_version < 12: - fnames = [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' - ] - - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) - + # >= 15 else: node.safe_psql( - "backupdb", - "CREATE SCHEMA ptrack") - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - node.safe_psql( - "backupdb", - "GRANT USAGE ON SCHEMA ptrack TO backup") + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) + + node.safe_psql( + "backupdb", + "CREATE SCHEMA ptrack") + node.safe_psql( + "backupdb", + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + node.safe_psql( + "backupdb", + "GRANT USAGE ON SCHEMA ptrack TO backup") node.safe_psql( "backupdb", @@ -244,11 +645,8 @@ def test_ptrack_unprivileged(self): if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") - - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') self.backup_node( backup_dir, 'node', node, @@ -263,10 +661,9 @@ def test_ptrack_unprivileged(self): # @unittest.expectedFailure def test_ptrack_enable(self): """make ptrack without full backup, should result in error""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', @@ -276,10 +673,9 @@ def test_ptrack_enable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # PTRACK BACKUP try: @@ -303,9 +699,6 @@ def test_ptrack_enable(self): ' CMD: {1}'.format(repr(e.message), self.cmd) ) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_disable(self): @@ -314,10 +707,9 @@ def test_ptrack_disable(self): enable ptrack, restart postgresql, take ptrack backup which should fail """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], @@ -327,28 +719,21 @@ def test_ptrack_disable(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) # DISABLE PTRACK - if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack.map_size to 0") - else: - node.safe_psql('postgres', "alter system set ptrack_enable to off") + node.safe_psql('postgres', "alter system set ptrack.map_size to 0") node.stop() node.slow_start() # ENABLE PTRACK - if node.major_version >= 12: - node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") - node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") - else: - node.safe_psql('postgres', "alter system set ptrack_enable to on") + node.safe_psql('postgres', "alter system set ptrack.map_size to '128'") + node.safe_psql('postgres', "alter system set shared_preload_libraries to 'ptrack'") node.stop() node.slow_start() @@ -376,31 +761,25 @@ def test_ptrack_disable(self): ) ) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_uncommitted_xact(self): """make ptrack backup while there is uncommitted open transaction""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'wal_level': 'replica', - 'autovacuum': 'off'}) + 'wal_level': 'replica'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -417,7 +796,7 @@ def test_ptrack_uncommitted_xact(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -437,17 +816,15 @@ def test_ptrack_uncommitted_xact(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_vacuum_full(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -458,10 +835,9 @@ def test_ptrack_vacuum_full(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'node', node, options=['--stream']) @@ -501,7 +877,7 @@ def test_ptrack_vacuum_full(self): process.join() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -524,23 +900,18 @@ def test_ptrack_vacuum_full(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_vacuum_truncate(self): """make node, create table, take full backup, delete last 3 pages, vacuum relation, take ptrack backup, take second ptrack backup, restore last ptrack backup and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -548,10 +919,9 @@ def test_ptrack_vacuum_truncate(self): self.create_tblspace_in_node(node, 'somedata') - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.safe_psql( "postgres", @@ -585,7 +955,7 @@ def test_ptrack_vacuum_truncate(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') @@ -610,17 +980,17 @@ def test_ptrack_vacuum_truncate(self): node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_get_block(self): - """make node, make full and ptrack stream backups," - " restore them and check data correctness""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """ + make node, make full and ptrack stream backups, + restore them and check data correctness + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -629,11 +999,9 @@ def test_ptrack_get_block(self): self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - self.skipTest("skip --- we do not need ptrack_get_block for ptrack 2.*") - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.safe_psql( "postgres", @@ -646,10 +1014,7 @@ def test_ptrack_get_block(self): options=['--stream'], gdb=True) - if node.major_version > 11: - gdb.set_breakpoint('make_pagemap_from_ptrack_2') - else: - gdb.set_breakpoint('make_pagemap_from_ptrack_1') + gdb.set_breakpoint('make_pagemap_from_ptrack_2') gdb.run_until_break() node.safe_psql( @@ -665,7 +1030,7 @@ def test_ptrack_get_block(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - result = node.safe_psql("postgres", "SELECT * FROM t_heap") + result = node.table_checksum("t_heap") node.cleanup() self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) @@ -679,36 +1044,28 @@ def test_ptrack_get_block(self): # Logical comparison self.assertEqual( result, - node.safe_psql("postgres", "SELECT * FROM t_heap") - ) - - # Clean after yourself - self.del_test_dir(module_name, fname) + node.table_checksum("t_heap")) # @unittest.skip("skip") def test_ptrack_stream(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql("postgres", "create sequence t_seq") @@ -718,7 +1075,7 @@ def test_ptrack_stream(self): " as t_seq, md5(i::text) as text, md5(i::text)::tsvector" " as tsvector from generate_series(0,100) i") - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node( backup_dir, 'node', node, options=['--stream']) @@ -729,7 +1086,7 @@ def test_ptrack_stream(self): " md5(i::text) as text, md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i") - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result = node.table_checksum("t_heap") ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) @@ -751,7 +1108,7 @@ def test_ptrack_stream(self): repr(self.output), self.cmd) ) node.slow_start() - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -771,37 +1128,30 @@ def test_ptrack_stream(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result_new = node.table_checksum("t_heap") self.assertEqual(ptrack_result, ptrack_result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_archive(self): """make archive node, make full and ptrack backups, check data correctness in restored instance""" - self.maxDiff = None - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -812,7 +1162,7 @@ def test_ptrack_archive(self): " md5(i::text)::tsvector as tsvector" " from generate_series(0,100) i") - full_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result = node.table_checksum("t_heap") full_backup_id = self.backup_node(backup_dir, 'node', node) full_target_time = self.show_pb( backup_dir, 'node', full_backup_id)['recovery-time'] @@ -825,7 +1175,7 @@ def test_ptrack_archive(self): " md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i") - ptrack_result = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result = node.table_checksum("t_heap") ptrack_backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack') ptrack_target_time = self.show_pb( @@ -833,6 +1183,13 @@ def test_ptrack_archive(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + node.safe_psql( + "postgres", + "insert into t_heap select i as id," + " md5(i::text) as text," + " md5(i::text)::tsvector as tsvector" + " from generate_series(200, 300) i") + # Drop Node node.cleanup() @@ -851,7 +1208,7 @@ def test_ptrack_archive(self): ) node.slow_start() - full_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() @@ -876,32 +1233,26 @@ def test_ptrack_archive(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() - ptrack_result_new = node.safe_psql("postgres", "SELECT * FROM t_heap") + ptrack_result_new = node.table_checksum("t_heap") self.assertEqual(ptrack_result, ptrack_result_new) node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + @unittest.skip("skip") def test_ptrack_pgpro417(self): - """Make node, take full backup, take ptrack backup, - delete ptrack backup. Try to take ptrack backup, - which should fail. Actual only for PTRACK 1.x""" - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + """ + Make node, take full backup, take ptrack backup, + delete ptrack backup. Try to take ptrack backup, + which should fail. Actual only for PTRACK 1.x + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -912,9 +1263,6 @@ def test_ptrack_pgpro417(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql( - "postgres", - "SELECT * FROM t_heap") backup_id = self.backup_node( backup_dir, 'node', node, @@ -929,7 +1277,7 @@ def test_ptrack_pgpro417(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=["--stream"]) @@ -962,29 +1310,21 @@ def test_ptrack_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + @unittest.skip("skip") def test_page_pgpro417(self): """ Make archive node, take full backup, take page backup, delete page backup. Try to take ptrack backup, which should fail. Actual only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -996,7 +1336,7 @@ def test_page_pgpro417(self): "postgres", "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") # PAGE BACKUP node.safe_psql( @@ -1004,7 +1344,7 @@ def test_page_pgpro417(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1033,29 +1373,21 @@ def test_page_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + @unittest.skip("skip") def test_full_pgpro417(self): """ Make node, take two full backups, delete full second backup. Try to take ptrack backup, which should fail. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1068,7 +1400,7 @@ def test_full_pgpro417(self): " md5(i::text)::tsvector as tsvector " " from generate_series(0,100) i" ) - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node(backup_dir, 'node', node, options=["--stream"]) # SECOND FULL BACKUP @@ -1078,7 +1410,7 @@ def test_full_pgpro417(self): " md5(i::text)::tsvector as tsvector" " from generate_series(100,200) i" ) - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1110,34 +1442,28 @@ def test_full_pgpro417(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_create_db(self): """ Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'max_wal_size': '10GB', - 'autovacuum': 'off'}) + 'max_wal_size': '10GB'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -1145,7 +1471,7 @@ def test_create_db(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.safe_psql("postgres", "SELECT * FROM t_heap") + node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, options=["--stream"]) @@ -1167,7 +1493,7 @@ def test_create_db(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1228,9 +1554,6 @@ def test_create_db(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_create_db_on_replica(self): """ @@ -1239,25 +1562,22 @@ def test_create_db_on_replica(self): create database db1, take ptrack backup from replica, restore database and check it presense """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP node.safe_psql( @@ -1266,7 +1586,7 @@ def test_create_db_on_replica(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node( @@ -1319,7 +1639,7 @@ def test_create_db_on_replica(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1332,32 +1652,26 @@ def test_create_db_on_replica(self): node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_table_set_tablespace_ptrack(self): """Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database.""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.create_tblspace_in_node(node, 'somedata') @@ -1377,8 +1691,7 @@ def test_alter_table_set_tablespace_ptrack(self): # sys.exit(1) # PTRACK BACKUP - #result = node.safe_psql( - # "postgres", "select * from t_heap") + #result = node.table_checksum("t_heap") self.backup_node( backup_dir, 'node', node, backup_type='ptrack', @@ -1391,7 +1704,7 @@ def test_alter_table_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -1420,38 +1733,31 @@ def test_alter_table_set_tablespace_ptrack(self): node_restored, {'port': node_restored.port}) node_restored.slow_start() -# result_new = node_restored.safe_psql( -# "postgres", "select * from t_heap") +# result_new = node_restored.table_checksum("t_heap") # # self.assertEqual(result, result_new, 'lost some data after restore') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_alter_database_set_tablespace_ptrack(self): """Make node, create tablespace with database," " take full backup, alter tablespace location," " take ptrack backup, restore database.""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1475,7 +1781,7 @@ def test_alter_database_set_tablespace_ptrack(self): # RESTORE node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( backup_dir, 'node', @@ -1496,34 +1802,28 @@ def test_alter_database_set_tablespace_ptrack(self): node_restored.port = node.port node_restored.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_drop_tablespace(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -1533,7 +1833,7 @@ def test_drop_tablespace(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1587,40 +1887,34 @@ def test_drop_tablespace(self): "Expecting Error because " "tablespace 'somedata' should not be present") - result_new = node.safe_psql("postgres", "select * from t_heap") + result_new = node.table_checksum("t_heap") self.assertEqual(result, result_new) if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_alter_tablespace(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') tblspc_path = self.get_tblspace_path(node, 'somedata') @@ -1631,7 +1925,7 @@ def test_ptrack_alter_tablespace(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=["--stream"]) @@ -1640,7 +1934,7 @@ def test_ptrack_alter_tablespace(self): "postgres", "alter table t_heap set tablespace somedata") # GET LOGICAL CONTENT FROM NODE - result = node.safe_psql("postgres", "select * from t_heap") + result = node.table_checksum("t_heap") # FIRTS PTRACK BACKUP self.backup_node( @@ -1653,7 +1947,7 @@ def test_ptrack_alter_tablespace(self): # Restore ptrack backup restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') @@ -1672,8 +1966,7 @@ def test_ptrack_alter_tablespace(self): restored_node.slow_start() # COMPARE LOGICAL CONTENT - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") + result_new = restored_node.table_checksum("t_heap") self.assertEqual(result, result_new) restored_node.cleanup() @@ -1707,48 +2000,42 @@ def test_ptrack_alter_tablespace(self): restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", "select * from t_heap") + result_new = restored_node.table_checksum("t_heap") self.assertEqual(result, result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_multiple_segments(self): """ Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'off'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') # CREATE TABLE node.pgbench_init(scale=100, options=['--tablespace=somedata']) + result = node.table_checksum("pgbench_accounts") # FULL BACKUP self.backup_node(backup_dir, 'node', node, options=['--stream']) # PTRACK STUFF - if node.major_version < 12: + if node.major_version < 11: idx_ptrack = {'type': 'heap'} idx_ptrack['path'] = self.get_fork_path(node, 'pgbench_accounts') idx_ptrack['old_size'] = self.get_fork_size(node, 'pgbench_accounts') @@ -1761,7 +2048,7 @@ def test_ptrack_multiple_segments(self): node.safe_psql("postgres", "checkpoint") - if node.major_version < 12: + if node.major_version < 11: idx_ptrack['new_size'] = self.get_fork_size( node, 'pgbench_accounts') @@ -1780,7 +2067,7 @@ def test_ptrack_multiple_segments(self): # GET LOGICAL CONTENT FROM NODE # it`s stupid, because hint`s are ignored by ptrack - result = node.safe_psql("postgres", "select * from pgbench_accounts") + result = node.table_checksum("pgbench_accounts") # FIRTS PTRACK BACKUP self.backup_node( backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) @@ -1790,7 +2077,7 @@ def test_ptrack_multiple_segments(self): # RESTORE NODE restored_node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'restored_node')) + base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( @@ -1813,9 +2100,7 @@ def test_ptrack_multiple_segments(self): restored_node, {'port': restored_node.port}) restored_node.slow_start() - result_new = restored_node.safe_psql( - "postgres", - "select * from pgbench_accounts") + result_new = restored_node.table_checksum("pgbench_accounts") # COMPARE RESTORED FILES self.assertEqual(result, result_new, 'data is lost') @@ -1823,29 +2108,21 @@ def test_ptrack_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_atexit_fail(self): """ Take backups of every available types and check that PTRACK is clean. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'max_connections': '15'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1880,27 +2157,20 @@ def test_atexit_fail(self): "select * from pg_is_in_backup()").rstrip(), "f") - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_clean(self): """ Take backups of every available types and check that PTRACK is clean Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1992,30 +2262,22 @@ def test_ptrack_clean(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_ptrack_clean_replica(self): """ Take backups of every available types from master and check that PTRACK on replica is clean. Relevant only for PTRACK 1.x """ - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() @@ -2023,7 +2285,7 @@ def test_ptrack_clean_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2135,27 +2397,23 @@ def test_ptrack_clean_replica(self): # check that ptrack bits are cleaned self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - # Clean after yourself - self.del_test_dir(module_name, fname) # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_on_btree(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2179,7 +2437,7 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -2197,30 +2455,25 @@ def test_ptrack_cluster_on_btree(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_ptrack_cluster_on_gist(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table and indexes node.safe_psql( @@ -2258,26 +2511,35 @@ def test_ptrack_cluster_on_gist(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - self.check_ptrack_map_sanity(node, idx_ptrack) + if node.major_version < 11: + self.check_ptrack_map_sanity(node, idx_ptrack) + + self.backup_node( + backup_dir, 'node', node, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) + self.restore_node(backup_dir, 'node', node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_ptrack_cluster_on_btree_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2285,7 +2547,7 @@ def test_ptrack_cluster_on_btree_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2339,25 +2601,37 @@ def test_ptrack_cluster_on_btree_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - self.check_ptrack_map_sanity(replica, idx_ptrack) + if master.major_version < 11: + self.check_ptrack_map_sanity(replica, idx_ptrack) + + self.backup_node( + backup_dir, 'replica', replica, + backup_type='ptrack', options=['-j10', '--stream']) + + pgdata = self.pgdata_content(replica.data_dir) + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node')) + node.cleanup() + + self.restore_node(backup_dir, 'replica', node) - # Clean after yourself - self.del_test_dir(module_name, fname) + pgdata_restored = self.pgdata_content(replica.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_ptrack_cluster_on_gist_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2365,7 +2639,7 @@ def test_ptrack_cluster_on_gist_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2417,13 +2691,13 @@ def test_ptrack_cluster_on_gist_replica(self): master.safe_psql('postgres', 'DELETE FROM t_heap WHERE id%2 = 1') master.safe_psql('postgres', 'CLUSTER t_heap USING t_gist') - if master.major_version < 12: + if master.major_version < 11: master.safe_psql('postgres', 'CHECKPOINT') # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) - if master.major_version < 12: + if master.major_version < 11: replica.safe_psql('postgres', 'CHECKPOINT') self.check_ptrack_map_sanity(replica, idx_ptrack) @@ -2435,7 +2709,7 @@ def test_ptrack_cluster_on_gist_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node) @@ -2444,31 +2718,24 @@ def test_ptrack_cluster_on_gist_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2499,7 +2766,7 @@ def test_ptrack_empty(self): node.safe_psql('postgres', 'checkpoint') node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') @@ -2524,9 +2791,6 @@ def test_ptrack_empty(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_empty_replica(self): @@ -2534,21 +2798,18 @@ def test_ptrack_empty_replica(self): Take backups of every available types from master and check that PTRACK on replica is clean """ - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], - ptrack_enable=True, - pg_options={ - 'autovacuum': 'off'}) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2556,7 +2817,7 @@ def test_ptrack_empty_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2612,7 +2873,7 @@ def test_ptrack_empty_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2623,28 +2884,23 @@ def test_ptrack_empty_replica(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_truncate(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2657,7 +2913,7 @@ def test_ptrack_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': node.safe_psql( @@ -2673,7 +2929,7 @@ def test_ptrack_truncate(self): node.safe_psql('postgres', 'truncate t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -2690,7 +2946,7 @@ def test_ptrack_truncate(self): pgdata = self.pgdata_content(node.data_dir) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( node, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -2706,28 +2962,24 @@ def test_ptrack_truncate(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") - def test_ptrack_truncate_replica(self): - fname = self.id().split('.')[3] + def test_basic_ptrack_truncate_replica(self): master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '32MB', 'archive_timeout': '10s', - 'checkpoint_timeout': '30s'}) + 'checkpoint_timeout': '5min'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2735,7 +2987,7 @@ def test_ptrack_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2765,7 +3017,7 @@ def test_ptrack_truncate_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -2785,7 +3037,7 @@ def test_ptrack_truncate_replica(self): '--master-db=postgres', '--master-port={0}'.format(master.port)]) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( replica, idx_ptrack[i]['path'], [idx_ptrack[i]['old_size']]) @@ -2795,7 +3047,15 @@ def test_ptrack_truncate_replica(self): # Sync master and replica self.wait_until_replica_catch_with_master(master, replica) - replica.safe_psql('postgres', 'checkpoint') + + if replica.major_version < 10: + replica.safe_psql( + "postgres", + "select pg_xlog_replay_pause()") + else: + replica.safe_psql( + "postgres", + "select pg_wal_replay_pause()") self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', @@ -2809,36 +3069,41 @@ def test_ptrack_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + if self.paranoia: + self.compare_pgdata(pgdata, pgdata_restored) + + self.set_auto_conf(node, {'port': node.port}) + + node.slow_start() + + node.safe_psql( + 'postgres', + 'select 1') # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -2860,6 +3125,8 @@ def test_ptrack_vacuum(self): idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) + node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') @@ -2867,7 +3134,7 @@ def test_ptrack_vacuum(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -2886,7 +3153,7 @@ def test_ptrack_vacuum(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -2903,28 +3170,24 @@ def test_ptrack_vacuum(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # @unittest.skip("skip") def test_ptrack_vacuum_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -2932,7 +3195,7 @@ def test_ptrack_vacuum_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -2972,7 +3235,7 @@ def test_ptrack_vacuum_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get fork size and calculate it in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -2995,7 +3258,7 @@ def test_ptrack_vacuum_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3005,7 +3268,7 @@ def test_ptrack_vacuum_replica(self): pgdata = self.pgdata_content(replica.data_dir) node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) @@ -3013,28 +3276,23 @@ def test_ptrack_vacuum_replica(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3056,6 +3314,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') self.backup_node( @@ -3064,7 +3323,7 @@ def test_ptrack_vacuum_bits_frozen(self): node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3075,7 +3334,7 @@ def test_ptrack_vacuum_bits_frozen(self): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3091,26 +3350,22 @@ def test_ptrack_vacuum_bits_frozen(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # @unittest.skip("skip") def test_ptrack_vacuum_bits_frozen_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3118,7 +3373,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3159,7 +3414,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3177,7 +3432,7 @@ def test_ptrack_vacuum_bits_frozen_replica(self): replica.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3192,28 +3447,23 @@ def test_ptrack_vacuum_bits_frozen_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3235,12 +3485,13 @@ def test_ptrack_vacuum_bits_visibility(self): i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) + comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3254,7 +3505,7 @@ def test_ptrack_vacuum_bits_visibility(self): node.safe_psql('postgres', 'checkpoint') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3270,29 +3521,25 @@ def test_ptrack_vacuum_bits_visibility(self): self.restore_node(backup_dir, 'node', node) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) + self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_vacuum_full(self): - fname = self.id().split('.')[3] + def test_ptrack_vacuum_full_2(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - ptrack_enable=True) + ptrack_enable=True, + pg_options={ 'wal_log_hints': 'on' }) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") self.create_tblspace_in_node(node, 'somedata') @@ -3318,7 +3565,7 @@ def test_ptrack_vacuum_full(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3332,7 +3579,7 @@ def test_ptrack_vacuum_full(self): node.safe_psql('postgres', 'vacuum full t_heap') node.safe_psql('postgres', 'checkpoint') - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3351,33 +3598,28 @@ def test_ptrack_vacuum_full(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3394,7 +3636,7 @@ def test_ptrack_vacuum_full_replica(self): "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as " "tsvector from generate_series(0,256000) i") - if master.major_version < 12: + if master.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': master.safe_psql( @@ -3421,7 +3663,7 @@ def test_ptrack_vacuum_full_replica(self): '--master-port={0}'.format(master.port), '--stream']) - if replica.major_version < 12: + if replica.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3439,7 +3681,7 @@ def test_ptrack_vacuum_full_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - if replica.major_version < 12: + if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3454,28 +3696,23 @@ def test_ptrack_vacuum_full_replica(self): pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure - def test_ptrack_vacuum_truncate(self): - fname = self.id().split('.')[3] + def test_ptrack_vacuum_truncate_2(self): node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table and indexes res = node.safe_psql( @@ -3486,7 +3723,7 @@ def test_ptrack_vacuum_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': node.safe_psql( @@ -3499,7 +3736,7 @@ def test_ptrack_vacuum_truncate(self): self.backup_node( backup_dir, 'node', node, options=['-j10', '--stream']) - if node.major_version < 12: + if node.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(node, i) @@ -3514,7 +3751,7 @@ def test_ptrack_vacuum_truncate(self): node.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - if node.major_version < 12: + if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) self.backup_node( @@ -3524,7 +3761,7 @@ def test_ptrack_vacuum_truncate(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3532,25 +3769,21 @@ def test_ptrack_vacuum_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_replica(self): - fname = self.id().split('.')[3] master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) master.slow_start() - if master.major_version >= 12: + if master.major_version >= 11: master.safe_psql( "postgres", "CREATE EXTENSION ptrack") @@ -3558,7 +3791,7 @@ def test_ptrack_vacuum_truncate_replica(self): self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -3598,7 +3831,7 @@ def test_ptrack_vacuum_truncate_replica(self): ] ) - if master.major_version < 12: + if master.major_version < 11: for i in idx_ptrack: # get size of heap and indexes. size calculated in pages idx_ptrack[i]['old_size'] = self.get_fork_size(replica, i) @@ -3617,7 +3850,7 @@ def test_ptrack_vacuum_truncate_replica(self): replica.safe_psql('postgres', 'CHECKPOINT') # CHECK PTRACK SANITY - if master.major_version < 12: + if master.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) self.backup_node( @@ -3630,7 +3863,7 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'replica', node_restored) @@ -3638,25 +3871,19 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - # @unittest.expectedFailure + @unittest.skip("skip") def test_ptrack_recovery(self): - if self.pg_config_version > self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL =< 11 for this test') - - fname = self.id().split('.')[3] + """ + Check that ptrack map contain correct bits after recovery. + Actual only for PTRACK 1.x + """ node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3702,27 +3929,19 @@ def test_ptrack_recovery(self): # check that ptrack has correct bits after recovery self.check_ptrack_recovery(idx_ptrack[i]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery_1(self): - if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3762,9 +3981,9 @@ def test_ptrack_recovery_1(self): 'postgres', "create extension pg_buffercache") - print(node.safe_psql( - 'postgres', - "SELECT count(*) FROM pg_buffercache WHERE isdirty")) + #print(node.safe_psql( + # 'postgres', + # "SELECT count(*) FROM pg_buffercache WHERE isdirty")) if self.verbose: print('Killing postmaster. Losing Ptrack changes') @@ -3783,7 +4002,7 @@ def test_ptrack_recovery_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -3792,28 +4011,23 @@ def test_ptrack_recovery_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_zero_changes(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table node.safe_psql( @@ -3838,32 +4052,26 @@ def test_ptrack_zero_changes(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Create table node.safe_psql( @@ -3948,14 +4156,14 @@ def test_ptrack_pg_resetxlog(self): except ProbackupException as e: self.assertTrue( 'ERROR: LSN from ptrack_control ' in e.message and - 'differs from Start LSN of previous backup' in e.message, + 'is greater than Start LSN of previous backup' in e.message, '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) # pgdata = self.pgdata_content(node.data_dir) # # node_restored = self.make_simple_node( -# base_dir=os.path.join(module_name, fname, 'node_restored')) +# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) # node_restored.cleanup() # # self.restore_node( @@ -3964,32 +4172,25 @@ def test_ptrack_pg_resetxlog(self): # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_ptrack_map(self): - - if self.pg_config_version < self.version_to_num('12.0'): - return unittest.skip('You need PostgreSQL >= 12 for this test') - - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + ptrack_version = self.get_ptrack_version(node) # Create table node.safe_psql( @@ -4015,48 +4216,55 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) ptrack_map = os.path.join(node.data_dir, 'global', 'ptrack.map') - ptrack_map_mmap = os.path.join(node.data_dir, 'global', 'ptrack.map.mmap') - # Let`s do index corruption. ptrack.map, ptrack.map.mmap + # Let`s do index corruption. ptrack.map with open(ptrack_map, "rb+", 0) as f: f.seek(42) f.write(b"blablahblahs") f.flush() f.close - with open(ptrack_map_mmap, "rb+", 0) as f: - f.seek(42) - f.write(b"blablahblahs") - f.flush() - f.close - # os.remove(os.path.join(node.logs_dir, node.pg_log_name)) - try: + if self.verbose: + print('Ptrack version:', ptrack_version) + if ptrack_version >= self.version_to_num("2.3"): node.slow_start() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack.map is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except StartNodeException as e: + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertIn( - 'Cannot start node', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + 'WARNING: ptrack read map: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() + node.stop(['-D', node.data_dir]) + else: + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because ptrack.map is corrupted" + "\n Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n' + ' CMD: {1}'.format(repr(e.message), self.cmd)) + + log_file = os.path.join(node.logs_dir, 'postgresql.log') + with open(log_file, 'r') as f: + log_content = f.read() - self.assertIn( - 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), - log_content) + self.assertIn( + 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), + log_content) self.set_auto_conf(node, {'ptrack.map_size': '0'}) - node.slow_start() try: @@ -4084,11 +4292,8 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) - node.slow_start() - sleep(1) - try: self.backup_node( backup_dir, 'node', node, @@ -4106,8 +4311,6 @@ def test_corrupt_ptrack_map(self): '\n Unexpected Error Message: {0}\n' ' CMD: {1}'.format(repr(e.message), self.cmd)) - sleep(1) - self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--stream']) @@ -4130,5 +4333,65 @@ def test_corrupt_ptrack_map(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_horizon_lsn_ptrack(self): + """ + https://github.com/postgrespro/pg_probackup/pull/386 + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + self.assertLessEqual( + self.version_to_num(self.old_probackup_version), + self.version_to_num('2.4.15'), + 'You need pg_probackup old_binary =< 2.4.15 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + self.assertGreaterEqual( + self.get_ptrack_version(node), + self.version_to_num("2.1"), + "You need ptrack >=2.1 for this test") + + # set map_size to a minimal value + self.set_auto_conf(node, {'ptrack.map_size': '1'}) + node.restart() + + node.pgbench_init(scale=100) + + # FULL backup + full_id = self.backup_node(backup_dir, 'node', node, options=['--stream'], old_binary=True) + + # enable archiving so the WAL size to do interfere with data bytes comparison later + self.set_archiving(backup_dir, 'node', node) + node.restart() + + # change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA is exemplar + delta_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') + delta_bytes = self.show_pb(backup_dir, 'node', backup_id=delta_id)["data-bytes"] + self.delete_pb(backup_dir, 'node', backup_id=delta_id) + + # PTRACK with current binary + ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + ptrack_bytes = self.show_pb(backup_dir, 'node', backup_id=ptrack_id)["data-bytes"] + + # make sure that backup size is exactly the same + self.assertEqual(delta_bytes, ptrack_bytes) diff --git a/tests/remote.py b/tests/remote_test.py similarity index 83% rename from tests/remote.py rename to tests/remote_test.py index 4d46447f0..2d36d7346 100644 --- a/tests/remote.py +++ b/tests/remote_test.py @@ -5,21 +5,17 @@ from .helpers.cfs_helpers import find_by_name -module_name = 'remote' - - class RemoteTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_remote_sanity(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -45,6 +41,3 @@ def test_remote_sanity(self): # e.message, # "\n Unexpected Error Message: {0}\n CMD: {1}".format( # repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/replica.py b/tests/replica_test.py similarity index 77% rename from tests/replica.py rename to tests/replica_test.py index de00c195c..17fc5a823 100644 --- a/tests/replica.py +++ b/tests/replica_test.py @@ -9,10 +9,85 @@ from time import sleep -module_name = 'replica' - class ReplicaTest(ProbackupTest, unittest.TestCase): + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_replica_switchover(self): + """ + check that archiving on replica works correctly + over the course of several switchovers + https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node1'), + set_replication=True, + initdb_params=['--data-checksums']) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node1', node1) + + node1.slow_start() + + # take full backup and restore it + self.backup_node(backup_dir, 'node1', node1, options=['--stream']) + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + # create replica + self.restore_node(backup_dir, 'node1', node2) + + # setup replica + self.add_instance(backup_dir, 'node2', node2) + self.set_archiving(backup_dir, 'node2', node2, replica=True) + self.set_replica(node1, node2, synchronous=False) + self.set_auto_conf(node2, {'port': node2.port}) + + node2.slow_start(replica=True) + + # generate some data + node1.pgbench_init(scale=5) + + # take full backup on replica + self.backup_node(backup_dir, 'node2', node2, options=['--stream']) + + # first switchover + node1.stop() + node2.promote() + + self.set_replica(node2, node1, synchronous=False) + node2.reload() + node1.slow_start(replica=True) + + # take incremental backup from new master + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # second switchover + node2.stop() + node1.promote() + self.set_replica(node1, node2, synchronous=False) + node1.reload() + node2.slow_start(replica=True) + + # generate some more data + node1.pgbench_init(scale=5) + + # take incremental backup from replica + self.backup_node( + backup_dir, 'node2', node2, + backup_type='delta', options=['--stream']) + + # https://github.com/postgrespro/pg_probackup/issues/251 + self.validate_pb(backup_dir) + # @unittest.skip("skip") # @unittest.expectedFailure def test_replica_stream_ptrack_backup(self): @@ -21,16 +96,15 @@ def test_replica_stream_ptrack_backup(self): take full stream backup from replica """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') if self.pg_config_version > self.version_to_num('9.6.0'): - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) @@ -51,19 +125,19 @@ def test_replica_stream_ptrack_backup(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") # take full backup and restore it self.backup_node(backup_dir, 'master', master, options=['--stream']) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) self.set_replica(master, replica) # Check data correctness on replica replica.slow_start(replica=True) - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -71,10 +145,10 @@ def test_replica_stream_ptrack_backup(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.add_instance(backup_dir, 'replica', replica) backup_id = self.backup_node( @@ -90,7 +164,7 @@ def test_replica_stream_ptrack_backup(self): # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -99,7 +173,7 @@ def test_replica_stream_ptrack_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take PTRACK backup from replica, @@ -107,11 +181,11 @@ def test_replica_stream_ptrack_backup(self): # to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'replica', replica, backup_type='ptrack', @@ -134,22 +208,18 @@ def test_replica_stream_ptrack_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_archive_page_backup(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -158,8 +228,7 @@ def test_replica_archive_page_backup(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -168,7 +237,7 @@ def test_replica_archive_page_backup(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -179,7 +248,7 @@ def test_replica_archive_page_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') @@ -193,7 +262,7 @@ def test_replica_archive_page_backup(self): replica.slow_start(replica=True) # Check data correctness on replica - after = replica.safe_psql("postgres", "SELECT * FROM t_heap") + after = replica.table_checksum("t_heap") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, @@ -201,11 +270,11 @@ def test_replica_archive_page_backup(self): # equal to original data master.psql( "postgres", - "insert into t_heap as select i as id, md5(i::text) as text, " + "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,25120) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") self.wait_until_replica_catch_with_master(master, replica) @@ -223,7 +292,7 @@ def test_replica_archive_page_backup(self): # RESTORE FULL BACKUP TAKEN FROM replica node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node')) + base_dir=os.path.join(self.module_name, self.fname, 'node')) node.cleanup() self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) @@ -232,7 +301,7 @@ def test_replica_archive_page_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM t_heap") + after = node.table_checksum("t_heap") self.assertEqual(before, after) node.cleanup() @@ -257,7 +326,7 @@ def test_replica_archive_page_backup(self): self.switch_wal_segment(master) - before = master.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + before = master.table_checksum("pgbench_accounts") self.validate_pb(backup_dir, 'replica') self.assertEqual( @@ -273,7 +342,7 @@ def test_replica_archive_page_backup(self): node.slow_start() # CHECK DATA CORRECTNESS - after = node.safe_psql("postgres", "SELECT * FROM pgbench_accounts") + after = master.table_checksum("pgbench_accounts") self.assertEqual( before, after, 'Restored data is not equal to original') @@ -281,27 +350,22 @@ def test_replica_archive_page_backup(self): self.backup_node( backup_dir, 'node', node, options=['--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_make_replica_via_restore(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -310,7 +374,7 @@ def test_basic_make_replica_via_restore(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -321,7 +385,7 @@ def test_basic_make_replica_via_restore(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,8192) i") - before = master.safe_psql("postgres", "SELECT * FROM t_heap") + before = master.table_checksum("t_heap") backup_id = self.backup_node( backup_dir, 'master', master, backup_type='page') @@ -339,9 +403,6 @@ def test_basic_make_replica_via_restore(self): backup_dir, 'replica', replica, options=['--archive-timeout=30s', '--stream']) - # Clean after yourself - self.del_test_dir(module_name, fname, [master, replica]) - # @unittest.skip("skip") def test_take_backup_from_delayed_replica(self): """ @@ -349,17 +410,15 @@ def test_take_backup_from_delayed_replica(self): restore full backup as delayed replica, launch pgbench, take FULL, PAGE and DELTA backups from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'archive_timeout': '10s'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -368,7 +427,7 @@ def test_take_backup_from_delayed_replica(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -448,19 +507,17 @@ def test_take_backup_from_delayed_replica(self): pgbench.wait() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote(self): """ start backup from replica, during backup promote replica check that backup is failed """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -469,8 +526,7 @@ def test_replica_promote(self): 'max_wal_size': '32MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -479,7 +535,7 @@ def test_replica_promote(self): master.slow_start() replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.backup_node(backup_dir, 'master', master) @@ -540,17 +596,15 @@ def test_replica_promote(self): 'setting its status to ERROR'.format(backup_id), log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_stop_lsn_null_offset(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -558,8 +612,7 @@ def test_replica_stop_lsn_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -575,7 +628,7 @@ def test_replica_stop_lsn_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -623,16 +676,17 @@ def test_replica_stop_lsn_null_offset(self): output) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb_checkpointer.kill() # @unittest.skip("skip") def test_replica_stop_lsn_null_offset_next_record(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -640,8 +694,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -651,13 +704,12 @@ def test_replica_stop_lsn_null_offset_next_record(self): # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) self.backup_node(backup_dir, 'master', master) # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -687,6 +739,7 @@ def test_replica_stop_lsn_null_offset_next_record(self): '--stream'], gdb=True) + # Attention! this breakpoint is set to a probackup internal function, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() @@ -718,22 +771,20 @@ def test_replica_stop_lsn_null_offset_next_record(self): log_content) self.assertIn( - 'LOG: stop_lsn: 0/4000000', + 'INFO: stop_lsn: 0/4000000', log_content) self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_replica_null_offset(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -741,8 +792,7 @@ def test_archive_replica_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -754,7 +804,7 @@ def test_archive_replica_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -807,17 +857,13 @@ def test_archive_replica_null_offset(self): print(output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_archive_replica_not_null_offset(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -825,8 +871,7 @@ def test_archive_replica_not_null_offset(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -838,7 +883,7 @@ def test_archive_replica_not_null_offset(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'node', replica) @@ -871,15 +916,17 @@ def test_archive_replica_not_null_offset(self): "\n Output: {0} \n CMD: {1}".format( repr(self.output), self.cmd)) except ProbackupException as e: - self.assertIn( - 'LOG: Looking for LSN 0/4000060 in segment: 000000010000000000000004', + # vanilla -- 0/4000060 + # pgproee -- 0/4000078 + self.assertRegex( e.message, + r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Wait for LSN 0/4000060 in archived WAL segment', + self.assertRegex( e.message, + r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -889,30 +936,26 @@ def test_archive_replica_not_null_offset(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_toast(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'checkpoint_timeout': '1h', 'wal_level': 'replica', 'shared_buffers': '128MB'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -928,7 +971,7 @@ def test_replica_toast(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -993,16 +1036,90 @@ def test_replica_toast(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - self.del_test_dir(module_name, fname) + gdb_checkpointer.kill() + + # @unittest.skip("skip") + def test_start_stop_lsn_in_the_same_segno(self): + """ + """ + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + master = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'master'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={ + 'checkpoint_timeout': '1h', + 'wal_level': 'replica', + 'shared_buffers': '128MB'}) + + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'master', master) + master.slow_start() + + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + + self.backup_node(backup_dir, 'master', master, options=['--stream']) + + # Create replica + replica = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica.cleanup() + self.restore_node(backup_dir, 'master', replica) + + # Settings for Replica + self.add_instance(backup_dir, 'replica', replica) + self.set_replica(master, replica, synchronous=True) + + replica.slow_start(replica=True) + + self.switch_wal_segment(master) + self.switch_wal_segment(master) + + master.safe_psql( + 'postgres', + 'CREATE TABLE t1 AS ' + 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'FROM generate_series(0,10) i') + + master.safe_psql( + 'postgres', + 'CHECKPOINT') + + self.wait_until_replica_catch_with_master(master, replica) + + sleep(60) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) + + self.backup_node( + backup_dir, 'replica', replica, + options=[ + '--archive-timeout=30', + '--log-level-console=LOG', + '--no-validate', + '--stream'], + return_id=False) @unittest.skip("skip") def test_replica_promote_1(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ @@ -1010,8 +1127,7 @@ def test_replica_promote_1(self): 'wal_level': 'replica'}) if self.get_version(master) < self.version_to_num('9.6.0'): - self.del_test_dir(module_name, fname) - return unittest.skip( + self.skipTest( 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) @@ -1024,7 +1140,7 @@ def test_replica_promote_1(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1067,17 +1183,13 @@ def test_replica_promote_1(self): os.path.exists(wal_file_partial), "File {0} disappeared".format(wal_file_partial)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_2(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) @@ -1092,7 +1204,7 @@ def test_replica_promote_2(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1116,88 +1228,6 @@ def test_replica_promote_2(self): backup_dir, 'master', replica, data_dir=replica.data_dir, backup_type='page') - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") - def test_replica_promote_3(self): - """ - """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) - replica.cleanup() - self.restore_node(backup_dir, 'master', replica) - - # Settings for Replica - self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) - - replica.slow_start(replica=True) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.add_instance(backup_dir, 'replica', replica) - - full_id = self.backup_node( - backup_dir, 'replica', - replica, options=['--stream']) - - master.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') - self.wait_until_replica_catch_with_master(master, replica) - - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - - replica.promote() - - # failing, because without archving, it is impossible to - # take multi-timeline backup. - try: - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of timeline switch " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Cannot find valid backup on previous timelines, ' - 'WAL archive is not available' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_archive_delta(self): """ @@ -1205,16 +1235,18 @@ def test_replica_promote_archive_delta(self): t2 /-------> t1 --F---D1--D2-- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'archive_timeout': '30s', - 'autovacuum': 'off'}) + 'archive_timeout': '30s'}) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) @@ -1228,7 +1260,7 @@ def test_replica_promote_archive_delta(self): # Create replica node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2, node2.data_dir) @@ -1316,9 +1348,6 @@ def test_replica_promote_archive_delta(self): pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_replica_promote_archive_page(self): """ @@ -1326,16 +1355,18 @@ def test_replica_promote_archive_page(self): t2 /-------> t1 --F---P1--P2-- """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', - 'archive_timeout': '30s', - 'autovacuum': 'off'}) + 'archive_timeout': '30s'}) + + if self.get_version(node1) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node1) @@ -1349,7 +1380,7 @@ def test_replica_promote_archive_page(self): # Create replica node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() self.restore_node(backup_dir, 'node', node2, node2.data_dir) @@ -1440,20 +1471,20 @@ def test_replica_promote_archive_page(self): pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_parent_choosing(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') master = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'master'), + base_dir=os.path.join(self.module_name, self.fname, 'master'), set_replication=True, initdb_params=['--data-checksums']) + if self.get_version(master) < self.version_to_num('9.6.0'): + self.skipTest( + 'Skipped because backup from replica is not supported in PG 9.5') + self.init_pb(backup_dir) self.add_instance(backup_dir, 'master', master) @@ -1463,7 +1494,7 @@ def test_parent_choosing(self): # Create replica replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() self.restore_node(backup_dir, 'master', replica) @@ -1501,35 +1532,17 @@ def test_parent_choosing(self): # failing, because without archving, it is impossible to # take multi-timeline backup. - try: - self.backup_node( - backup_dir, 'replica', replica, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of timeline switch " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Cannot find valid backup on previous timelines, ' - 'WAL archive is not available' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - # Clean after yourself - self.del_test_dir(module_name, fname) + self.backup_node( + backup_dir, 'replica', replica, + backup_type='delta', options=['--stream']) # @unittest.skip("skip") def test_instance_from_the_past(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1566,8 +1579,75 @@ def test_instance_from_the_past(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_replica_via_basebackup(self): + """ + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={'hot_standby': 'on'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + + node.slow_start() + + node.pgbench_init(scale=10) + + #FULL backup + full_id = self.backup_node(backup_dir, 'node', node) + + pgbench = node.pgbench( + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + node.cleanup() + + self.restore_node( + backup_dir, 'node', node, + options=['--recovery-target=latest', '--recovery-target-action=promote']) + node.slow_start() + + # Timeline 2 + # Take stream page backup from instance in timeline2 + self.backup_node( + backup_dir, 'node', node, backup_type='full', + options=['--stream', '--log-level-file=verbose']) + + node.cleanup() + + # restore stream backup + self.restore_node(backup_dir, 'node', node) + + xlog_dir = 'pg_wal' + if self.get_version(node) < 100000: + xlog_dir = 'pg_xlog' + + filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") + self.assertTrue( + os.path.exists(filepath), + "History file do not exists: {0}".format(filepath)) + + node.slow_start() + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored.cleanup() + + pg_basebackup_path = self.get_bin_path('pg_basebackup') + + self.run_binary( + [ + pg_basebackup_path, '-p', str(node.port), '-h', 'localhost', + '-R', '-X', 'stream', '-D', node_restored.data_dir + ]) + + self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.slow_start(replica=True) # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..e2ac18bea --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,13 @@ +# Testgres can be installed in the following ways: +# 1. From a pip package (recommended) +# testgres==1.8.5 +# 2. From a specific Git branch, tag or commit +# git+https://github.com/postgrespro/testgres.git@ +# 3. From a local directory +# /path/to/local/directory/testgres +git+https://github.com/postgrespro/testgres.git@archive-command-exec#egg=testgres-pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 +allure-pytest +deprecation +pexpect +pytest==7.4.3 +pytest-xdist diff --git a/tests/restore.py b/tests/restore_test.py similarity index 71% rename from tests/restore.py rename to tests/restore_test.py index 9c105175e..b6664252e 100644 --- a/tests/restore.py +++ b/tests/restore_test.py @@ -2,17 +2,15 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException import subprocess -from datetime import datetime import sys -from time import sleep -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone import hashlib import shutil import json -from testgres import QueryException - - -module_name = 'restore' +import stat +from shutil import copyfile +from testgres import QueryException, StartNodeException +from stat import S_ISDIR class RestoreTest(ProbackupTest, unittest.TestCase): @@ -21,12 +19,11 @@ class RestoreTest(ProbackupTest, unittest.TestCase): # @unittest.expectedFailure def test_restore_full_to_latest(self): """recovery to latest from full backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -37,7 +34,7 @@ def test_restore_full_to_latest(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) node.stop() @@ -52,29 +49,28 @@ def test_restore_full_to_latest(self): repr(self.output), self.cmd)) # 2 - Test that recovery.conf was created + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') self.assertEqual(os.path.isfile(recovery_conf), True) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -92,7 +88,7 @@ def test_restore_full_page_to_latest(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="page") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -106,21 +102,17 @@ def test_restore_full_page_to_latest(self): node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_specific_timeline(self): """recovery to target timeline""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -128,7 +120,7 @@ def test_restore_to_specific_timeline(self): node.pgbench_init(scale=2) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) @@ -172,33 +164,31 @@ def test_restore_to_specific_timeline(self): self.assertEqual(int(recovery_target_timeline), target_tli) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_time(self): """recovery to target time""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], - pg_options={'TimeZone': 'Europe/Moscow'}) + pg_options={'TimeZone': 'GMT'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() node.pgbench_init(scale=2) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") backup_id = self.backup_node(backup_dir, 'node', node) - target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + target_time = node.execute( + "postgres", "SELECT to_char(now(), 'YYYY-MM-DD HH24:MI:SS+00')" + )[0][0] pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -220,21 +210,17 @@ def test_restore_to_time(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): """recovery to target xid""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -252,7 +238,7 @@ def test_restore_to_xid_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: res = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() @@ -278,23 +264,19 @@ def test_restore_to_xid_inclusive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_xid_not_inclusive(self): """recovery with target inclusive false""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -312,7 +294,7 @@ def test_restore_to_xid_not_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: result = con.execute("INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") con.commit() @@ -339,27 +321,22 @@ def test_restore_to_xid_not_inclusive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 0) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_lsn_inclusive(self): """recovery to target lsn""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(module_name, fname) return - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -377,7 +354,7 @@ def test_restore_to_lsn_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() @@ -410,27 +387,22 @@ def test_restore_to_lsn_inclusive(self): node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_to_lsn_not_inclusive(self): """recovery to target lsn""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < self.version_to_num('10.0'): - self.del_test_dir(module_name, fname) return - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -448,7 +420,7 @@ def test_restore_to_lsn_not_inclusive(self): pgbench.wait() pgbench.stdout.close() - before = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() @@ -482,36 +454,31 @@ def test_restore_to_lsn_not_inclusive(self): node.slow_start() - after = node.safe_psql("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) self.assertEqual( len(node.execute("postgres", "SELECT * FROM tbl0005")), 1) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_archive(self): """recovery to latest from archive full+ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -525,7 +492,7 @@ def test_restore_full_ptrack_archive(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="ptrack") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -539,34 +506,29 @@ def test_restore_full_ptrack_archive(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_ptrack(self): """recovery to latest from archive full+ptrack+ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -587,7 +549,7 @@ def test_restore_ptrack(self): backup_id = self.backup_node( backup_dir, 'node', node, backup_type="ptrack") - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -601,35 +563,30 @@ def test_restore_ptrack(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_stream(self): """recovery in stream mode to latest from full + ptrack backups""" if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -644,7 +601,7 @@ def test_restore_full_ptrack_stream(self): backup_dir, 'node', node, backup_type="ptrack", options=["--stream"]) - before = node.execute("postgres", "SELECT * FROM pgbench_branches") + before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() @@ -657,12 +614,9 @@ def test_restore_full_ptrack_stream(self): repr(self.output), self.cmd)) node.slow_start() - after = node.execute("postgres", "SELECT * FROM pgbench_branches") + after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_ptrack_under_load(self): """ @@ -670,25 +624,23 @@ def test_restore_full_ptrack_under_load(self): with loads when ptrack backup do """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") node.pgbench_init(scale=2) @@ -730,9 +682,6 @@ def test_restore_full_ptrack_under_load(self): "postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_full_under_load_ptrack(self): """ @@ -740,25 +689,23 @@ def test_restore_full_under_load_ptrack(self): with loads when full backup do """ if not self.ptrack: - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() - if node.major_version >= 12: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # wal_segment_size = self.guc_wal_segment_size(node) node.pgbench_init(scale=2) @@ -802,18 +749,14 @@ def test_restore_full_under_load_ptrack(self): "postgres", "SELECT sum(delta) FROM pgbench_history") self.assertEqual(bbalance, delta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -865,7 +808,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore tablespace destination is not empty:', + 'ERROR: Restore tablespace destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -888,7 +831,7 @@ def test_restore_with_tablespace_mapping_1(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: restore tablespace destination is not empty:', + 'ERROR: Restore tablespace destination is not empty:', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -939,18 +882,14 @@ def test_restore_with_tablespace_mapping_1(self): result = node.execute("postgres", "SELECT id FROM test OFFSET 1") self.assertEqual(result[0][0], 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1016,8 +955,131 @@ def test_restore_with_tablespace_mapping_2(self): count = node.execute("postgres", "SELECT count(*) FROM tbl1") self.assertEqual(count[0][0], 4) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_restore_with_missing_or_corrupted_tablespace_map(self): + """restore backup with missing or corrupted tablespace_map""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + self.create_tblspace_in_node(node, 'tblspace') + node.pgbench_init(scale=1, tablespace='tblspace') + + # Full backup + self.backup_node(backup_dir, 'node', node) + + # Change some data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Page backup + page_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + node2 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2.cleanup() + + olddir = self.get_tblspace_path(node, 'tblspace') + newdir = self.get_tblspace_path(node2, 'tblspace') + + # drop tablespace_map + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map') + + tablespace_map_tmp = os.path.join( + backup_dir, 'backups', 'node', + page_id, 'database', 'tablespace_map_tmp') + + os.rename(tablespace_map, tablespace_map_tmp) + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is missing.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Tablespace map is missing: "{0}", ' + 'probably backup {1} is corrupt, validate it'.format( + tablespace_map, page_id), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + copyfile(tablespace_map_tmp, tablespace_map) + + with open(tablespace_map, "a") as f: + f.write("HELLO\n") + + try: + self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)]) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + try: + self.restore_node(backup_dir, 'node', node2) + self.assertEqual( + 1, 0, + "Expecting Error because tablespace_map is corupted.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertIn( + 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # rename it back + os.rename(tablespace_map_tmp, tablespace_map) + + print(self.restore_node( + backup_dir, 'node', node2, + options=["-T", "{0}={1}".format(olddir, newdir)])) + + pgdata_restored = self.pgdata_content(node2.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_archive_node_backup_stream_restore_to_recovery_time(self): @@ -1025,13 +1087,12 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): make node with archiving, make stream backup, make PITR to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1064,9 +1125,6 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_stream_restore_to_recovery_time(self): @@ -1074,13 +1132,12 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): make node with archiving, make stream backup, make PITR to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1111,9 +1168,6 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): result = node.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_stream_pitr(self): @@ -1122,13 +1176,12 @@ def test_archive_node_backup_stream_pitr(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1159,9 +1212,6 @@ def test_archive_node_backup_stream_pitr(self): result = node.psql("postgres", 'select * from t_heap') self.assertEqual(True, 'does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_node_backup_archive_pitr_2(self): @@ -1170,12 +1220,11 @@ def test_archive_node_backup_archive_pitr_2(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1189,7 +1238,7 @@ def test_archive_node_backup_archive_pitr_2(self): node.stop() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() recovery_time = self.show_pb( @@ -1217,9 +1266,6 @@ def test_archive_node_backup_archive_pitr_2(self): result = node_restored.psql("postgres", 'select * from t_heap') self.assertTrue('does not exist' in result[2].decode("utf-8")) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_archive_restore_to_restore_point(self): @@ -1228,12 +1274,11 @@ def test_archive_restore_to_restore_point(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1244,9 +1289,7 @@ def test_archive_restore_to_restore_point(self): node.safe_psql( "postgres", "create table t_heap as select generate_series(0,10000)") - result = node.safe_psql( - "postgres", - "select * from t_heap") + result = node.table_checksum("t_heap") node.safe_psql( "postgres", "select pg_create_restore_point('savepoint')") node.safe_psql( @@ -1262,7 +1305,7 @@ def test_archive_restore_to_restore_point(self): node.slow_start() - result_new = node.safe_psql("postgres", "select * from t_heap") + result_new = node.table_checksum("t_heap") res = node.psql("postgres", "select * from t_heap_1") self.assertEqual( res[0], 1, @@ -1270,18 +1313,14 @@ def test_archive_restore_to_restore_point(self): self.assertEqual(result, result_new) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1332,7 +1371,7 @@ def test_zags_block_corrupt(self): "insert into tbl select i from generate_series(0,100) as i") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), initdb_params=['--data-checksums']) node_restored.cleanup() @@ -1349,15 +1388,13 @@ def test_zags_block_corrupt(self): @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt_1(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums'], pg_options={ - 'autovacuum': 'off', 'full_page_writes': 'on'} ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1408,7 +1445,7 @@ def test_zags_block_corrupt_1(self): self.switch_wal_segment(node) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored'), + base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), initdb_params=['--data-checksums']) pgdata = self.pgdata_content(node.data_dir) @@ -1458,13 +1495,12 @@ def test_restore_chain(self): ERROR delta backups, take valid delta backup, restore must be successfull """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1482,7 +1518,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1490,7 +1526,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1502,7 +1538,7 @@ def test_restore_chain(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1540,19 +1576,15 @@ def test_restore_chain(self): self.restore_node(backup_dir, 'node', node) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_chain_with_corrupted_backup(self): """more complex test_restore_chain()""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1570,7 +1602,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=0s']) + backup_type='page', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1582,7 +1614,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1594,7 +1626,7 @@ def test_restore_chain_with_corrupted_backup(self): try: self.backup_node( backup_dir, 'node', node, - backup_type='delta', options=['--archive-timeout=0s']) + backup_type='delta', options=['-U', 'wrong_name']) except ProbackupException as e: pass @@ -1720,21 +1752,19 @@ def test_restore_chain_with_corrupted_backup(self): node.cleanup() - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + # Skipped, because backups from the future are invalid. + # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" + # now (PBCKP-259). We can conduct such a test again when we + # untie 'backup_id' from 'start_time' + @unittest.skip("skip") def test_restore_backup_from_future(self): """more complex test_restore_chain()""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'autovacuum': 'off'}) + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1777,22 +1807,18 @@ def test_restore_backup_from_future(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_immediate_stream(self): """ correct handling of immediate recovery target for STREAM backups """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1808,8 +1834,11 @@ def test_restore_target_immediate_stream(self): pgdata = self.pgdata_content(node.data_dir) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1831,22 +1860,18 @@ def test_restore_target_immediate_stream(self): os.path.isfile(recovery_conf), "File {0} do not exists".format(recovery_conf)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_target_immediate_archive(self): """ correct handling of immediate recovery target for ARCHIVE backups """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1863,8 +1888,11 @@ def test_restore_target_immediate_archive(self): pgdata = self.pgdata_content(node.data_dir) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1888,22 +1916,20 @@ def test_restore_target_immediate_archive(self): with open(recovery_conf, 'r') as f: self.assertIn("recovery_target = 'immediate'", f.read()) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.skip("skip") + # Skipped, because default recovery_target_timeline is 'current' + # Before PBCKP-598 the --recovery-target=latest' option did not work and this test allways passed + @unittest.skip("skip") def test_restore_target_latest_archive(self): """ make sure that recovery_target 'latest' is default recovery target """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1913,7 +1939,7 @@ def test_restore_target_latest_archive(self): self.backup_node(backup_dir, 'node', node) if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1925,25 +1951,34 @@ def test_restore_target_latest_archive(self): # open(recovery_conf, 'rb').read()).hexdigest() with open(recovery_conf, 'r') as f: - content_1 = f.read() + content_1 = '' + while True: + line = f.readline() - # restore - node.cleanup() + if not line: + break + if line.startswith("#"): + continue + content_1 += line + node.cleanup() self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) # hash_2 = hashlib.md5( # open(recovery_conf, 'rb').read()).hexdigest() with open(recovery_conf, 'r') as f: - content_2 = f.read() - - self.assertEqual(content_1, content_2) + content_2 = '' + while True: + line = f.readline() - # self.assertEqual(hash_1, hash_2) + if not line: + break + if line.startswith("#"): + continue + content_2 += line - # Clean after yourself - self.del_test_dir(module_name, fname) + self.assertEqual(content_1, content_2) # @unittest.skip("skip") def test_restore_target_new_options(self): @@ -1951,13 +1986,12 @@ def test_restore_target_new_options(self): check that new --recovery-target-* options are working correctly """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1966,8 +2000,11 @@ def test_restore_target_new_options(self): # Take FULL self.backup_node(backup_dir, 'node', node) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1986,7 +2023,8 @@ def test_restore_target_new_options(self): target_name = 'savepoint' - target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + # in python-3.6+ it can be ...now()..astimezone()... + target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %z") with node.connect("postgres") as con: res = con.execute( "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") @@ -2118,9 +2156,6 @@ def test_restore_target_new_options(self): node.slow_start() - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_smart_restore(self): """ @@ -2130,13 +2165,12 @@ def test_smart_restore(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2182,26 +2216,22 @@ def test_smart_restore(self): for file in filelist_diff: self.assertNotIn(file, logfile_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pg_11_group_access(self): """ test group access for PG >= 11 """ if self.pg_config_version < self.version_to_num('11.0'): - return unittest.skip('You need PostgreSQL >= 11 for this test') + self.skipTest('You need PostgreSQL >= 11 for this test') - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=[ '--data-checksums', '--allow-group-access']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -2213,7 +2243,7 @@ def test_pg_11_group_access(self): # restore backup node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node( @@ -2223,16 +2253,14 @@ def test_pg_11_group_access(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_concurrent_drop_table(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2278,16 +2306,12 @@ def test_restore_concurrent_drop_table(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_lost_non_data_file(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2325,15 +2349,11 @@ def test_lost_non_data_file(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_exclude(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2349,7 +2369,7 @@ def test_partial_restore_exclude(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -2364,7 +2384,7 @@ def test_partial_restore_exclude(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() try: @@ -2401,7 +2421,7 @@ def test_partial_restore_exclude(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() self.restore_node( @@ -2440,15 +2460,11 @@ def test_partial_restore_exclude(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_exclude_tablespace(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2470,7 +2486,7 @@ def test_partial_restore_exclude_tablespace(self): 'postgres', "SELECT oid " "FROM pg_tablespace " - "WHERE spcname = 'somedata'").rstrip() + "WHERE spcname = 'somedata'").decode('utf-8').rstrip() for i in range(1, 10, 1): node.safe_psql( @@ -2480,7 +2496,7 @@ def test_partial_restore_exclude_tablespace(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -2495,7 +2511,7 @@ def test_partial_restore_exclude_tablespace(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') @@ -2521,7 +2537,7 @@ def test_partial_restore_exclude_tablespace(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') @@ -2563,16 +2579,12 @@ def test_partial_restore_exclude_tablespace(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_include(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -2588,7 +2600,7 @@ def test_partial_restore_include(self): db_list_raw = node.safe_psql( 'postgres', 'SELECT to_json(a) ' - 'FROM (SELECT oid, datname FROM pg_database) a').rstrip() + 'FROM (SELECT oid, datname FROM pg_database) a').decode('utf-8').rstrip() db_list_splitted = db_list_raw.splitlines() @@ -2603,7 +2615,7 @@ def test_partial_restore_include(self): # restore FULL backup node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() try: @@ -2642,7 +2654,7 @@ def test_partial_restore_include(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) node_restored_2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) node_restored_2.cleanup() self.restore_node( @@ -2689,17 +2701,16 @@ def test_partial_restore_include(self): self.assertNotIn('PANIC', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_partial_restore_backward_compatibility_1(self): """ old binary should be of version < 2.2.0 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2722,7 +2733,7 @@ def test_partial_restore_backward_compatibility_1(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -2782,7 +2793,7 @@ def test_partial_restore_backward_compatibility_1(self): # get new node node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() self.restore_node( @@ -2799,10 +2810,12 @@ def test_partial_restore_backward_compatibility_merge(self): """ old binary should be of version < 2.2.0 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2825,7 +2838,7 @@ def test_partial_restore_backward_compatibility_merge(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -2885,7 +2898,7 @@ def test_partial_restore_backward_compatibility_merge(self): # get new node node_restored_1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored_1')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) node_restored_1.cleanup() # merge @@ -2903,10 +2916,9 @@ def test_partial_restore_backward_compatibility_merge(self): def test_empty_and_mangled_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2934,7 +2946,7 @@ def test_empty_and_mangled_database_map(self): f.close() node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() try: @@ -2985,7 +2997,7 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' + 'ERROR: Field "dbOid" is not found in the line 42 of ' 'the file backup_content.control', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3001,7 +3013,7 @@ def test_empty_and_mangled_database_map(self): self.output, self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: field "dbOid" is not found in the line 42 of ' + 'ERROR: Field "dbOid" is not found in the line 42 of ' 'the file backup_content.control', e.message, '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) @@ -3016,14 +3028,12 @@ def test_empty_and_mangled_database_map(self): def test_missing_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -3063,16 +3073,17 @@ def test_missing_database_map(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") # PG 9.6 elif self.get_version(node) > 90600 and self.get_version(node) < 100000: node.safe_psql( @@ -3094,11 +3105,14 @@ def test_missing_database_map(self): "GRANT CONNECT ON DATABASE backupdb to backup; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -3109,8 +3123,8 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - # >= 10 - else: + # >= 10 && < 15 + elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -3132,8 +3146,10 @@ def test_missing_database_map(self): "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " @@ -3144,43 +3160,61 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) + # >= 15 + else: + node.safe_psql( + 'backupdb', + "REVOKE ALL ON DATABASE backupdb from PUBLIC; " + "REVOKE ALL ON SCHEMA public from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " + "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " + "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " + "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " + "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " + "CREATE ROLE backup WITH LOGIN REPLICATION; " + "GRANT CONNECT ON DATABASE backupdb to backup; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " + "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack + "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" + ) if self.ptrack: - fnames = [] - if node.major_version < 12: - fnames += [ - 'pg_catalog.oideq(oid, oid)', - 'pg_catalog.ptrack_version()', - 'pg_catalog.pg_ptrack_clear()', - 'pg_catalog.pg_ptrack_control_lsn()', - 'pg_catalog.pg_ptrack_get_and_clear_db(oid, oid)', - 'pg_catalog.pg_ptrack_get_and_clear(oid, oid)', - 'pg_catalog.pg_ptrack_get_block_2(oid, oid, oid, bigint)' - ] - else: - # TODO why backup works without these grants ? -# fnames += [ -# 'pg_ptrack_get_pagemapset(pg_lsn)', -# 'pg_ptrack_control_lsn()', -# 'pg_ptrack_get_block(oid, oid, oid, bigint)' -# ] - node.safe_psql( - "backupdb", - "CREATE EXTENSION ptrack WITH SCHEMA pg_catalog") - - for fname in fnames: - node.safe_psql( - "backupdb", - "GRANT EXECUTE ON FUNCTION {0} TO backup".format(fname)) - - if ProbackupTest.enterprise: + # TODO why backup works without these grants ? + # 'pg_ptrack_get_pagemapset(pg_lsn)', + # 'pg_ptrack_control_lsn()', + # because PUBLIC node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup") + "CREATE SCHEMA ptrack; " + "GRANT USAGE ON SCHEMA ptrack TO backup; " + "CREATE EXTENSION ptrack WITH SCHEMA ptrack") + + if ProbackupTest.enterprise: node.safe_psql( "backupdb", - "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup") + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup without database_map backup_id = self.backup_node( @@ -3190,7 +3224,7 @@ def test_missing_database_map(self): pgdata = self.pgdata_content(node.data_dir) node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() # backup has missing database_map and that is legal @@ -3234,9 +3268,6 @@ def test_missing_database_map(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_stream_restore_command_option(self): """ @@ -3251,21 +3282,23 @@ def test_stream_restore_command_option(self): as replica, check that PostgreSQL recovery uses restore_command to obtain WAL from archive. """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={'max_wal_size': '32MB'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(node.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -3308,24 +3341,20 @@ def test_stream_restore_command_option(self): timeline_id = node.safe_psql( 'postgres', - 'select timeline_id from pg_control_checkpoint()').rstrip() + 'select timeline_id from pg_control_checkpoint()').decode('utf-8').rstrip() self.assertEqual('2', timeline_id) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_primary_conninfo(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3338,7 +3367,7 @@ def test_restore_primary_conninfo(self): #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' @@ -3352,8 +3381,11 @@ def test_restore_primary_conninfo(self): os.path.isfile(standby_signal), "File '{0}' do not exists".format(standby_signal)) + # TODO update test if self.get_version(node) >= self.version_to_num('12.0'): - recovery_conf = os.path.join(replica.data_dir, 'probackup_recovery.conf') + recovery_conf = os.path.join(replica.data_dir, 'postgresql.auto.conf') + with open(recovery_conf, 'r') as f: + print(f.read()) else: recovery_conf = os.path.join(replica.data_dir, 'recovery.conf') @@ -3362,20 +3394,16 @@ def test_restore_primary_conninfo(self): self.assertIn(str_conninfo, recovery_conf_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_restore_primary_slot_info(self): """ """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -3386,7 +3414,7 @@ def test_restore_primary_slot_info(self): node.pgbench_init(scale=1) replica = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'replica')) + base_dir=os.path.join(self.module_name, self.fname, 'replica')) replica.cleanup() node.safe_psql( @@ -3407,5 +3435,498 @@ def test_restore_primary_slot_info(self): replica.slow_start(replica=True) - # Clean after yourself - self.del_test_dir(module_name, fname) + def test_issue_249(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + 'postgres', + 'CREATE database db1') + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.safe_psql( + 'postgres', + 'select * from pgbench_accounts') + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + 'postgres', + 'INSERT INTO pgbench_accounts SELECT * FROM t1') + + # restore FULL backup + node_restored_1 = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1.cleanup() + + self.restore_node( + backup_dir, 'node', + node_restored_1, options=["--db-include=db1"]) + + self.set_auto_conf( + node_restored_1, + {'port': node_restored_1.port, 'hot_standby': 'off'}) + + node_restored_1.slow_start() + + node_restored_1.safe_psql( + 'db1', + 'select 1') + + try: + node_restored_1.safe_psql( + 'postgres', + 'select 1') + except QueryException as e: + self.assertIn('FATAL', e.message) + + def test_pg_12_probackup_recovery_conf_compatibility(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + if not self.probackup_old_path: + self.skipTest("You must specify PGPROBACKUPBIN_OLD" + " for run this test") + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): + self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + node.safe_psql( + 'postgres', + 'CREATE TABLE t1 as SELECT * from pgbench_accounts where aid > 200000 and aid < 450000') + + time = node.safe_psql( + 'SELECT current_timestamp(0)::timestamptz;').decode('utf-8').rstrip() + + node.safe_psql( + 'postgres', + 'DELETE from pgbench_accounts where aid > 200000 and aid < 450000') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-time={0}".format(time), + "--recovery-target-action=promote"], + old_binary=True) + + node.slow_start() + + self.backup_node(backup_dir, 'node', node, old_binary=True) + + node.pgbench_init(scale=5) + + xid = node.safe_psql( + 'SELECT txid_current()').decode('utf-8').rstrip() + node.pgbench_init(scale=1) + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target-xid={0}".format(xid), + "--recovery-target-action=promote"]) + + node.slow_start() + + def test_drop_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # drop postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + os.remove(auto_path) + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + def test_truncate_postgresql_auto_conf(self): + """ + https://github.com/postgrespro/pg_probackup/issues/249 + + pg_probackup version must be 12 or greater + """ + + if self.pg_config_version < self.version_to_num('12.0'): + self.skipTest('You need PostgreSQL >= 12 for this test') + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL backup + self.backup_node(backup_dir, 'node', node) + + # truncate postgresql.auto.conf + auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") + with open(auto_path, "w+") as f: + f.truncate() + + self.backup_node(backup_dir, 'node', node, backup_type='page') + + node.cleanup() + + self.restore_node( + backup_dir, 'node',node, + options=[ + "--recovery-target=latest", + "--recovery-target-action=promote"]) + node.slow_start() + + self.assertTrue(os.path.exists(auto_path)) + + # @unittest.skip("skip") + def test_concurrent_restore(self): + """""" + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL backup + self.backup_node( + backup_dir, 'node', node, + options=['--stream', '--compress']) + + pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # DELTA backup + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--stream', '--compress', '--no-validate']) + + pgdata1 = self.pgdata_content(node.data_dir) + + node_restored = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + + node.cleanup() + node_restored.cleanup() + + gdb = self.restore_node( + backup_dir, 'node', node, options=['--no-validate'], gdb=True) + + gdb.set_breakpoint('restore_data_file') + gdb.run_until_break() + + self.restore_node( + backup_dir, 'node', node_restored, options=['--no-validate']) + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + pgdata2 = self.pgdata_content(node.data_dir) + pgdata3 = self.pgdata_content(node_restored.data_dir) + + self.compare_pgdata(pgdata1, pgdata2) + self.compare_pgdata(pgdata2, pgdata3) + + # @unittest.skip("skip") + def test_restore_with_waldir(self): + """recovery using tablespace-mapping option and page backup""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + + with node.connect("postgres") as con: + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Full backup + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + # Create waldir + waldir_path = os.path.join(node.base_dir, "waldir") + os.makedirs(waldir_path) + + # Test recovery from latest + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, + options=[ + "-X", "%s" % (waldir_path)]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + + # check pg_wal is symlink + if node.major_version >= 10: + wal_path=os.path.join(node.data_dir, "pg_wal") + else: + wal_path=os.path.join(node.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + + # @unittest.skip("skip") + def test_restore_to_latest_timeline(self): + """recovery to latest timeline""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + node.pgbench_init(scale=2) + + before1 = node.table_checksum("pgbench_branches") + backup_id = self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), + self.restore_node( + backup_dir, 'node', node, options=["-j", "4"]), + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + before2 = node.table_checksum("pgbench_branches") + self.backup_node(backup_dir, 'node', node) + + node.stop() + node.cleanup() + # restore from first backup + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target-timeline=latest", "-i", backup_id] + ) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + # check recovery-target=latest option for compatibility with previous versions + node.cleanup() + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "--recovery-target=latest", "-i", backup_id] + ) + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + + # start postgres and promote wal files to latest timeline + node.slow_start() + + # check for the latest updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before2, after) + + # checking recovery_target_timeline=current is the default option + if self.pg_config_version >= self.version_to_num('12.0'): + node.stop() + node.cleanup() + + # restore from first backup + restore_result = self.restore_node(backup_dir, 'node', node, + options=[ + "-j", "4", "-i", backup_id] + ) + + self.assertIn( + "INFO: Restore of backup {0} completed.".format(backup_id), restore_result, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(self.output), self.cmd)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "current") + + # start postgres with current timeline + node.slow_start() + + # check for the current updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before1, after) + + def test_restore_issue_313(self): + """ + Check that partially restored PostgreSQL instance cannot be started + """ + self._check_gdb_flag_or_skip_test + node = self.make_simple_node('node', + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + # FULL backup + backup_id = self.backup_node(backup_dir, 'node', node) + node.cleanup() + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and \ + not stat.S_ISDIR(int(filelist[file]['mode'])) and \ + not filelist[file]['size'] == '0' and \ + file != 'database_map': + count += 1 + + node_restored = self.make_simple_node('node_restored') + node_restored.cleanup() + self.restore_node(backup_dir, 'node', node_restored) + + gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 1) + gdb.quit() + + # emulate the user or HA taking care of PG configuration + for fname in os.listdir(node_restored.data_dir): + if fname.endswith('.conf'): + os.rename( + os.path.join(node_restored.data_dir, fname), + os.path.join(node.data_dir, fname)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + if self.pg_config_version >= 120000: + self.assertIn( + "PANIC: could not read file \"global/pg_control\"", + f.read()) + else: + self.assertIn( + "PANIC: could not read from control file", + f.read()) diff --git a/tests/retention.py b/tests/retention_test.py similarity index 87% rename from tests/retention.py rename to tests/retention_test.py index 0d1c72b41..88432a00f 100644 --- a/tests/retention.py +++ b/tests/retention_test.py @@ -6,21 +6,17 @@ from distutils.dir_util import copy_tree -module_name = 'retention' - - class RetentionTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -41,7 +37,7 @@ def test_retention_redundancy_1(self): output_before = self.show_archive(backup_dir, 'node', tli=1) # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) @@ -71,18 +67,14 @@ def test_retention_redundancy_1(self): self.assertTrue(wal_name >= min_wal) self.assertTrue(wal_name <= max_wal) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -124,31 +116,27 @@ def test_retention_window_2(self): self.delete_expired(backup_dir, 'node', options=['--expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() # take FULL BACKUP - backup_id_1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take second FULL BACKUP - backup_id_2 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take third FULL BACKUP - backup_id_3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) backups = os.path.join(backup_dir, 'backups', 'node') for backup in os.listdir(backups): @@ -171,25 +159,21 @@ def test_retention_window_3(self): # count wal files in ARCHIVE - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_4(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() # take FULL BACKUPs - backup_id_1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) backup_id_2 = self.backup_node(backup_dir, 'node', node) @@ -232,18 +216,14 @@ def test_retention_window_4(self): n_wals = len(os.listdir(wals_dir)) self.assertTrue(n_wals == 0) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -356,18 +336,14 @@ def test_window_expire_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_redundancy_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -444,8 +420,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.backup_node(backup_dir, 'node', node, backup_type='page') # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -467,18 +442,14 @@ def test_redundancy_expire_interleaved_incremental_chains(self): print(self.show_pb( backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -604,9 +575,6 @@ def test_window_merge_interleaved_incremental_chains(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains_1(self): """ @@ -617,13 +585,11 @@ def test_window_merge_interleaved_incremental_chains_1(self): FULLb FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum':'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -632,7 +598,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): node.pgbench_init(scale=5) # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() @@ -663,13 +629,13 @@ def test_window_merge_interleaved_incremental_chains_1(self): # PAGEa1 ERROR # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) @@ -711,7 +677,7 @@ def test_window_merge_interleaved_incremental_chains_1(self): conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=3))) - output = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--retention-window=1', '--expired', '--merge-expired']) @@ -747,9 +713,6 @@ def test_window_merge_interleaved_incremental_chains_1(self): pgdata_restored_b3 = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata_b3, pgdata_restored_b3) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants(self): """ @@ -763,13 +726,11 @@ def test_basic_window_merge_multiple_descendants(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1008,9 +969,6 @@ def test_basic_window_merge_multiple_descendants(self): self.show_pb(backup_dir, 'node')[0]['backup-mode'], 'FULL') - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): """ @@ -1024,13 +982,11 @@ def test_basic_window_merge_multiple_descendants_1(self): FULLb | FULLa """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1275,9 +1231,6 @@ def test_basic_window_merge_multiple_descendants_1(self): '--retention-window=1', '--delete-expired', '--log-level-console=log']) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_window_chains(self): """ @@ -1290,13 +1243,11 @@ def test_window_chains(self): PAGE FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1305,26 +1256,26 @@ def test_window_chains(self): node.pgbench_init(scale=3) # Chain A - backup_id_a = self.backup_node(backup_dir, 'node', node) - page_id_a1 = self.backup_node( + self.backup_node(backup_dir, 'node', node) + self.backup_node( backup_dir, 'node', node, backup_type='page') - page_id_a2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Chain B - backup_id_b = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) @@ -1347,7 +1298,7 @@ def test_window_chains(self): conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( datetime.now() - timedelta(days=3))) - output = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=[ '--retention-window=1', '--expired', @@ -1362,9 +1313,6 @@ def test_window_chains(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_chains_1(self): """ @@ -1377,12 +1325,11 @@ def test_window_chains_1(self): PAGE FULL """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1391,26 +1338,26 @@ def test_window_chains_1(self): node.pgbench_init(scale=3) # Chain A - backup_id_a = self.backup_node(backup_dir, 'node', node) - page_id_a1 = self.backup_node( + self.backup_node(backup_dir, 'node', node) + self.backup_node( backup_dir, 'node', node, backup_type='page') - page_id_a2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Chain B - backup_id_b = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) - page_id_b1 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta') - page_id_b2 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') page_id_b3 = self.backup_node( backup_dir, 'node', node, backup_type='delta') - pgdata = self.pgdata_content(node.data_dir) + self.pgdata_content(node.data_dir) # Purge backups backups = os.path.join(backup_dir, 'backups', 'node') @@ -1456,9 +1403,6 @@ def test_window_chains_1(self): "Purging finished", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_window_error_backups(self): """ @@ -1471,30 +1415,26 @@ def test_window_error_backups(self): FULL -------redundancy """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() # Take FULL BACKUPs - backup_id_a1 = self.backup_node(backup_dir, 'node', node) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) + self.backup_node(backup_dir, 'node', node) + self.backup_node( + backup_dir, 'node', node, backup_type='page') - page_id_a3 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Change FULLb backup status to ERROR - self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - - # Clean after yourself - self.del_test_dir(module_name, fname) + # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # @unittest.skip("skip") def test_window_error_backups_1(self): @@ -1504,45 +1444,44 @@ def test_window_error_backups_1(self): FULL -------window """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() # Take FULL BACKUP - full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take PAGE BACKUP gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb.remove_all_breakpoints() gdb._execute('signal SIGINT') gdb.continue_execution_until_error() - page_id = self.show_pb(backup_dir, 'node')[1]['id'] + self.show_pb(backup_dir, 'node')[1]['id'] # Take DELTA backup - delta_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) # Take FULL BACKUP - full2_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_window_error_backups_2(self): """ @@ -1551,30 +1490,32 @@ def test_window_error_backups_2(self): FULL -------window """ - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) node.slow_start() # Take FULL BACKUP - full_id = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # Take PAGE BACKUP gdb = self.backup_node( backup_dir, 'node', node, backup_type='page', gdb=True) + # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() gdb._execute('signal SIGKILL') gdb.continue_execution_until_error() - page_id = self.show_pb(backup_dir, 'node')[1]['id'] + self.show_pb(backup_dir, 'node')[1]['id'] if self.get_version(node) < 90600: node.safe_psql( @@ -1582,27 +1523,24 @@ def test_window_error_backups_2(self): 'SELECT pg_catalog.pg_stop_backup()') # Take DELTA backup - delta_id = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) - # Clean after yourself - # self.del_test_dir(module_name, fname) - def test_retention_redundancy_overlapping_chains(self): """""" - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1630,27 +1568,24 @@ def test_retention_redundancy_overlapping_chains(self): self.backup_node(backup_dir, 'node', node, backup_type="page") # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - - def test_retention_redundancy_overlapping_chains(self): + def test_retention_redundancy_overlapping_chains_1(self): """""" - fname = self.id().split('.')[3] + self._check_gdb_flag_or_skip_test() + node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) if self.get_version(node) < 90600: - self.del_test_dir(module_name, fname) - return unittest.skip('Skipped because ptrack support is disabled') + self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1678,25 +1613,21 @@ def test_retention_redundancy_overlapping_chains(self): self.backup_node(backup_dir, 'node', node, backup_type="page") # Purge backups - log = self.delete_expired( + self.delete_expired( backup_dir, 'node', options=['--expired', '--wal']) self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) self.validate_pb(backup_dir, 'node') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_wal_purge_victim(self): """ https://github.com/postgrespro/pg_probackup/issues/103 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1713,7 +1644,7 @@ def test_wal_purge_victim(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertTrue( - "WARNING: Valid backup on current timeline 1 is not found" in e.message and + "WARNING: Valid full backup on current timeline 1 is not found" in e.message and "ERROR: Create new full backup before an incremental one" in e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -1741,19 +1672,17 @@ def test_wal_purge_victim(self): "WARNING: Backup {0} has missing parent 0".format(page_id), e.message) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_failed_merge_redundancy_retention(self): """ Check that retention purge works correctly with MERGING backups """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( base_dir=os.path.join( - module_name, fname, 'node'), + self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -1837,9 +1766,6 @@ def test_failed_merge_redundancy_retention(self): self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_wal_depth_1(self): """ |-------------B5----------> WAL timeline3 @@ -1848,16 +1774,14 @@ def test_wal_depth_1(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', - 'checkpoint_timeout': '30s', - 'autovacuum': 'off'}) + 'checkpoint_timeout': '30s'}) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -1869,7 +1793,7 @@ def test_wal_depth_1(self): # FULL node.pgbench_init(scale=1) - B1 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) # PAGE node.pgbench_init(scale=1) @@ -1881,21 +1805,21 @@ def test_wal_depth_1(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=1) - B3 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') node.pgbench_init(scale=1) - B4 = self.backup_node( + self.backup_node( backup_dir, 'node', node, backup_type='page') # Timeline 2 node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() @@ -1917,7 +1841,7 @@ def test_wal_depth_1(self): target_xid = node_restored.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_restored.pgbench_init(scale=2) @@ -1940,11 +1864,11 @@ def test_wal_depth_1(self): node_restored.slow_start() node_restored.pgbench_init(scale=1) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) node.pgbench_init(scale=1) - B6 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] @@ -1954,8 +1878,6 @@ def test_wal_depth_1(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_wal_purge(self): """ -------------------------------------> tli5 @@ -1976,10 +1898,9 @@ def test_wal_purge(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2003,25 +1924,27 @@ def test_wal_purge(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=5) # B2 FULL on TLI1 - B2 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) - B3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) + self.delete_pb(backup_dir, 'node', options=['--delete-wal']) + # TLI 2 node_tli2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) node_tli2.cleanup() output = self.restore_node( backup_dir, 'node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-timeline=1', '--recovery-target-action=promote']) self.assertIn( @@ -2034,21 +1957,21 @@ def test_wal_purge(self): target_xid = node_tli2.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) - B4 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() # TLI3 node_tli3 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli3')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) node_tli3.cleanup() # Note, that successful validation here is a happy coincidence @@ -2069,7 +1992,7 @@ def test_wal_purge(self): # TLI4 node_tli4 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli4')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) node_tli4.cleanup() self.restore_node( @@ -2084,14 +2007,14 @@ def test_wal_purge(self): node_tli4.pgbench_init(scale=5) - B6 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() # TLI5 node_tli5 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli5')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) node_tli5.cleanup() self.restore_node( @@ -2174,8 +2097,6 @@ def test_wal_purge(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_wal_depth_2(self): """ -------------------------------------> tli5 @@ -2197,10 +2118,9 @@ def test_wal_depth_2(self): wal-depth=2 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2224,25 +2144,25 @@ def test_wal_depth_2(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node.pgbench_init(scale=5) # B2 FULL on TLI1 B2 = self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) - B3 = self.backup_node(backup_dir, 'node', node) + self.backup_node(backup_dir, 'node', node) node.pgbench_init(scale=4) # TLI 2 node_tli2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli2')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) node_tli2.cleanup() output = self.restore_node( backup_dir, 'node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), - '--recovery-target-timeline=1'.format(target_xid), + '--recovery-target-timeline=1', '--recovery-target-action=promote']) self.assertIn( @@ -2255,21 +2175,21 @@ def test_wal_depth_2(self): target_xid = node_tli2.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) B4 = self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - B5 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() # TLI3 node_tli3 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli3')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) node_tli3.cleanup() # Note, that successful validation here is a happy coincidence @@ -2290,7 +2210,7 @@ def test_wal_depth_2(self): # TLI4 node_tli4 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli4')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) node_tli4.cleanup() self.restore_node( @@ -2305,14 +2225,14 @@ def test_wal_depth_2(self): node_tli4.pgbench_init(scale=5) - B6 = self.backup_node( + self.backup_node( backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() # TLI5 node_tli5 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_tli5')) + base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) node_tli5.cleanup() self.restore_node( @@ -2431,8 +2351,6 @@ def test_wal_depth_2(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname) - def test_basic_wal_depth(self): """ B1---B1----B3-----B4----B5------> tli1 @@ -2442,10 +2360,9 @@ def test_basic_wal_depth(self): wal-depth=1 """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -2490,7 +2407,7 @@ def test_basic_wal_depth(self): target_xid = node.safe_psql( "postgres", - "select txid_current()").rstrip() + "select txid_current()").decode('utf-8').rstrip() self.switch_wal_segment(node) @@ -2534,4 +2451,79 @@ def test_basic_wal_depth(self): self.validate_pb(backup_dir, 'node') - self.del_test_dir(module_name, fname, [node]) + def test_concurrent_running_full_backup(self): + """ + https://github.com/postgrespro/pg_probackup/issues/328 + """ + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # FULL + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'RUNNING') + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired']) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.backup_node(backup_dir, 'node', node) + + gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb.set_breakpoint('backup_data_file') + gdb.run_until_break() + gdb.kill() + + self.backup_node( + backup_dir, 'node', node, backup_type='delta', + options=['--retention-redundancy=2', '--delete-expired'], + return_id=False) + + self.assertTrue( + self.show_pb(backup_dir, 'node')[0]['status'], + 'OK') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[1]['status'], + 'RUNNING') + + self.assertTrue( + self.show_pb(backup_dir, 'node')[2]['status'], + 'OK') + + self.assertEqual( + len(self.show_pb(backup_dir, 'node')), + 6) diff --git a/tests/set_backup.py b/tests/set_backup_test.py similarity index 86% rename from tests/set_backup.py rename to tests/set_backup_test.py index db039c92d..31334cfba 100644 --- a/tests/set_backup.py +++ b/tests/set_backup_test.py @@ -5,8 +5,6 @@ from sys import exit from datetime import datetime, timedelta -module_name = 'set_backup' - class SetBackupTest(ProbackupTest, unittest.TestCase): @@ -14,10 +12,9 @@ class SetBackupTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") def test_set_backup_sanity(self): """general sanity for set-backup command""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -44,7 +41,7 @@ def test_set_backup_sanity(self): repr(self.output), self.cmd)) except ProbackupException as e: self.assertIn( - 'ERROR: required parameter not specified: --instance', + 'ERROR: Required parameter not specified: --instance', e.message, "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) @@ -120,19 +117,15 @@ def test_set_backup_sanity(self): # parse string to datetime object #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_pinning(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -174,18 +167,14 @@ def test_retention_redundancy_pinning(self): '{1} is guarded by retention'.format(full_id, page_id), log) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_retention_window_pinning(self): """purge all backups using window-based retention policy""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -237,9 +226,6 @@ def test_retention_window_pinning(self): '{1} is guarded by retention'.format(backup_id_1, page1), out) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_wal_retention_and_pinning(self): """ @@ -251,13 +237,12 @@ def test_wal_retention_and_pinning(self): B1 B2---P---B3---> """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -317,9 +302,6 @@ def test_wal_retention_and_pinning(self): '000000010000000000000004') self.assertEqual(timeline['status'], 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_wal_retention_and_pinning_1(self): """ @@ -331,12 +313,11 @@ def test_wal_retention_and_pinning_1(self): P---B1---> """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -383,19 +364,15 @@ def test_wal_retention_and_pinning_1(self): self.validate_pb(backup_dir) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_add_note_newlines(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -418,19 +395,15 @@ def test_add_note_newlines(self): backup_meta = self.show_pb(backup_dir, 'node', backup_id) self.assertNotIn('note', backup_meta) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_add_big_note(self): """""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -463,7 +436,7 @@ def test_add_big_note(self): note = node.safe_psql( "postgres", - "SELECT repeat('hello', 200)").rstrip() + "SELECT repeat('hello', 200)").decode('utf-8').rstrip() backup_id = self.backup_node( backup_dir, 'node', node, @@ -472,5 +445,32 @@ def test_add_big_note(self): backup_meta = self.show_pb(backup_dir, 'node', backup_id) self.assertEqual(backup_meta['note'], note) - # Clean after yourself - self.del_test_dir(module_name, fname) \ No newline at end of file + + # @unittest.skip("skip") + def test_add_big_note_1(self): + """""" + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + note = node.safe_psql( + "postgres", + "SELECT repeat('q', 1024)").decode('utf-8').rstrip() + + # FULL + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + + self.set_backup( + backup_dir, 'node', backup_id, + options=['--note={0}'.format(note)]) + + backup_meta = self.show_pb(backup_dir, 'node', backup_id) + + print(backup_meta) + self.assertEqual(backup_meta['note'], note) diff --git a/tests/show.py b/tests/show_test.py similarity index 71% rename from tests/show.py rename to tests/show_test.py index 0da95dcbb..27b6fab96 100644 --- a/tests/show.py +++ b/tests/show_test.py @@ -3,19 +3,15 @@ from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -module_name = 'show' - - -class OptionTest(ProbackupTest, unittest.TestCase): +class ShowTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") # @unittest.expectedFailure def test_show_1(self): """Status DONE and OK""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -31,17 +27,13 @@ def test_show_1(self): ) self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_show_json(self): """Status DONE and OK""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -58,16 +50,12 @@ def test_show_json(self): self.backup_node(backup_dir, 'node', node) self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -102,16 +90,12 @@ def test_corrupt_2(self): ) self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_no_control_file(self): """backup.control doesn't exist""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -137,16 +121,12 @@ def test_no_control_file(self): 'doesn\'t exist', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_empty_control_file(self): """backup.control is empty""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -173,17 +153,13 @@ def test_empty_control_file(self): 'is empty', output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_control_file(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -205,19 +181,17 @@ def test_corrupt_control_file(self): 'WARNING: Invalid option "statuss" in file', self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -233,12 +207,7 @@ def test_corrupt_correctness(self): output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - if self.remote: - backup_remote_id = self.backup_node(backup_dir, 'node', node) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node(backup_dir, 'node', node) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -261,13 +230,8 @@ def test_corrupt_correctness(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -291,13 +255,8 @@ def test_corrupt_correctness(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -312,19 +271,17 @@ def test_corrupt_correctness(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness_1(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -340,12 +297,7 @@ def test_corrupt_correctness_1(self): output_local = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_local_id) - if self.remote: - backup_remote_id = self.backup_node(backup_dir, 'node', node) - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node(backup_dir, 'node', node) output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -372,13 +324,8 @@ def test_corrupt_correctness_1(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='delta') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -402,13 +349,8 @@ def test_corrupt_correctness_1(self): backup_dir, 'node', as_json=False, backup_id=backup_local_id) self.delete_pb(backup_dir, 'node', backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--remote-proto=ssh', '--remote-host=localhost']) + backup_remote_id = self.backup_node( + backup_dir, 'node', node, backup_type='page') output_remote = self.show_pb( backup_dir, 'node', as_json=False, backup_id=backup_remote_id) @@ -423,19 +365,17 @@ def test_corrupt_correctness_1(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_correctness_2(self): """backup.control contains invalid option""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + if not self.remote: + self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" + " for run this test") + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums'], - pg_options={'autovacuum': 'off'}) + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -536,5 +476,70 @@ def test_corrupt_correctness_2(self): output_local['uncompressed-bytes'], output_remote['uncompressed-bytes']) - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + # @unittest.expectedFailure + def test_color_with_no_terminal(self): + """backup.control contains invalid option""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums'], + pg_options={'autovacuum': 'off'}) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + # FULL + try: + self.backup_node( + backup_dir, 'node', node, options=['--archive-timeout=1s']) + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because archiving is disabled\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertNotIn( + '[0m', e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + # @unittest.skip("skip") + def test_tablespace_print_issue_431(self): + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + full_backup_id = self.backup_node(backup_dir, 'node', node) + self.assertIn("OK", self.show_pb(backup_dir,'node', as_text=True)) + # Check that tablespace info exists. JSON + self.assertIn("tablespace_map", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn("oid", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn("path", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn(tblspc_path, self.show_pb(backup_dir, 'node', as_text=True)) + # Check that tablespace info exists. PLAIN + self.assertIn("tablespace_map", self.show_pb(backup_dir, 'node', backup_id=full_backup_id, as_text=True, as_json=False)) + self.assertIn(tblspc_path, self.show_pb(backup_dir, 'node', backup_id=full_backup_id, as_text=True, as_json=False)) + # Check that tablespace info NOT exists if backup id not provided. PLAIN + self.assertNotIn("tablespace_map", self.show_pb(backup_dir, 'node', as_text=True, as_json=False)) diff --git a/tests/snapfs.py b/tests/snapfs.py deleted file mode 100644 index a7f926c4c..000000000 --- a/tests/snapfs.py +++ /dev/null @@ -1,60 +0,0 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -module_name = 'snapfs' - - -class SnapFSTest(ProbackupTest, unittest.TestCase): - - # @unittest.expectedFailure - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_snapfs_simple(self): - """standart backup modes with ARCHIVE WAL method""" - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - 'postgres', - 'select pg_make_snapshot()') - - node.pgbench_init(scale=10) - - pgbench = node.pgbench(options=['-T', '50', '-c', '2', '--no-vacuum']) - pgbench.wait() - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - node.safe_psql( - 'postgres', - 'select pg_remove_snapshot(1)') - - self.backup_node( - backup_dir, 'node', node, backup_type='page') - - pgdata = self.pgdata_content(node.data_dir) - - node.cleanup() - - self.restore_node( - backup_dir, 'node', - node, options=["-j", "4"]) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/time_consuming_test.py b/tests/time_consuming_test.py new file mode 100644 index 000000000..c0038c085 --- /dev/null +++ b/tests/time_consuming_test.py @@ -0,0 +1,77 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest +import subprocess +from time import sleep + + +class TimeConsumingTests(ProbackupTest, unittest.TestCase): + def test_pbckp150(self): + """ + https://jira.postgrespro.ru/browse/PBCKP-150 + create a node filled with pgbench + create FULL backup followed by PTRACK backup + run pgbench, vacuum VERBOSE FULL and ptrack backups in parallel + """ + # init node + if self.pg_config_version < self.version_to_num('11.0'): + self.skipTest('You need PostgreSQL >= 11 for this test') + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + ptrack_enable=self.ptrack, + initdb_params=['--data-checksums'], + pg_options={ + 'max_connections': 100, + 'log_statement': 'none', + 'log_checkpoints': 'on', + 'autovacuum': 'off', + 'ptrack.map_size': 1}) + + if node.major_version >= 13: + self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) + else: + self.set_auto_conf(node, {'wal_keep_segments': '1000'}) + + # init probackup and add an instance + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + + # run the node and init ptrack + node.slow_start() + node.safe_psql("postgres", "CREATE EXTENSION ptrack") + # populate it with pgbench + node.pgbench_init(scale=5) + + # FULL backup followed by PTRACK backup + self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + + # run ordinary pgbench scenario to imitate some activity and another pgbench for vacuuming in parallel + nBenchDuration = 30 + pgbench = node.pgbench(options=['-c', '20', '-j', '8', '-T', str(nBenchDuration)]) + with open('/tmp/pbckp150vacuum.sql', 'w') as f: + f.write('VACUUM (FULL) pgbench_accounts, pgbench_tellers, pgbench_history; SELECT pg_sleep(1);\n') + pgbenchval = node.pgbench(options=['-c', '1', '-f', '/tmp/pbckp150vacuum.sql', '-T', str(nBenchDuration)]) + + # several PTRACK backups + for i in range(nBenchDuration): + print("[{}] backing up PTRACK diff...".format(i+1)) + self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) + sleep(0.1) + # if the activity pgbench has finished, stop backing up + if pgbench.poll() is not None: + break + + pgbench.kill() + pgbenchval.kill() + pgbench.wait() + pgbenchval.wait() + + backups = self.show_pb(backup_dir, 'node') + for b in backups: + self.assertEqual("OK", b['status']) diff --git a/tests/time_stamp.py b/tests/time_stamp.py deleted file mode 100644 index 8abd55a2b..000000000 --- a/tests/time_stamp.py +++ /dev/null @@ -1,74 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -module_name = 'time_stamp' - -class CheckTimeStamp(ProbackupTest, unittest.TestCase): - - def test_start_time_format(self): - """Test backup ID changing after start-time editing in backup.control. - We should convert local time in UTC format""" - # Create simple node - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums']) - - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.start() - - backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) - show_backup = self.show_pb(backup_dir, 'node') - - i = 0 - while i < 2: - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: - output = "" - for line in f: - if line.startswith('start-time') is True: - if i == 0: - output = output + str(line[:-5])+'+00\''+'\n' - else: - output = output + str(line[:-5]) + '\'' + '\n' - else: - output = output + str(line) - f.close() - - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: - fw.write(output) - fw.flush() - show_backup = show_backup + self.show_pb(backup_dir, 'node') - i += 1 - - self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") - - node.stop() - # Clean after yourself - self.del_test_dir(module_name, fname) - - def test_server_date_style(self): - """Issue #112""" - fname = self.id().split('.')[3] - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(module_name, fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={"datestyle": "GERMAN, DMY"}) - - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.start() - - self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j 2']) - - # Clean after yourself - self.del_test_dir(module_name, fname) diff --git a/tests/time_stamp_test.py b/tests/time_stamp_test.py new file mode 100644 index 000000000..170c62cd4 --- /dev/null +++ b/tests/time_stamp_test.py @@ -0,0 +1,236 @@ +import os +import unittest +from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import subprocess +from time import sleep + + +class TimeStamp(ProbackupTest, unittest.TestCase): + + def test_start_time_format(self): + """Test backup ID changing after start-time editing in backup.control. + We should convert local time in UTC format""" + # Create simple node + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.start() + + backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) + show_backup = self.show_pb(backup_dir, 'node') + + i = 0 + while i < 2: + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: + output = "" + for line in f: + if line.startswith('start-time') is True: + if i == 0: + output = output + str(line[:-5])+'+00\''+'\n' + else: + output = output + str(line[:-5]) + '\'' + '\n' + else: + output = output + str(line) + f.close() + + with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: + fw.write(output) + fw.flush() + show_backup = show_backup + self.show_pb(backup_dir, 'node') + i += 1 + + print(show_backup[1]['id']) + print(show_backup[2]['id']) + + self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + node.stop() + + def test_server_date_style(self): + """Issue #112""" + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums'], + pg_options={"datestyle": "GERMAN, DMY"}) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2']) + + def test_handling_of_TZ_env_variable(self): + """Issue #284""" + node = self.make_simple_node( + base_dir="{0}/{1}/node".format(self.module_name, self.fname), + set_replication=True, + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.start() + + my_env = os.environ.copy() + my_env["TZ"] = "America/Detroit" + + self.backup_node( + backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) + + output = self.show_pb(backup_dir, 'node', as_json=False, as_text=True, env=my_env) + + self.assertNotIn("backup ID in control file", output) + + @unittest.skip("skip") + # @unittest.expectedFailure + def test_dst_timezone_handling(self): + """for manual testing""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + print(subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + output = self.backup_node(backup_dir, 'node', node, return_id=False) + self.assertNotIn("backup ID in control file", output) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', return_id=False) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + @unittest.skip("skip") + def test_dst_timezone_handling_backward_compatibilty(self): + """for manual testing""" + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'America/Detroit'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'false'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-05-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # FULL + self.backup_node(backup_dir, 'node', node, old_binary=True, return_id=False) + + # move to dst + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-10-25 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + output = self.backup_node( + backup_dir, 'node', node, backup_type='delta', old_binary=True, return_id=False) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + # DELTA + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-ntp', 'true'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + + sleep(10) + + self.backup_node(backup_dir, 'node', node, backup_type='delta') + + output = self.show_pb(backup_dir, as_json=False, as_text=True) + self.assertNotIn("backup ID in control file", output) + + subprocess.Popen( + ['sudo', 'timedatectl', 'set-timezone', 'US/Moscow'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() diff --git a/tests/validate.py b/tests/validate_test.py similarity index 89% rename from tests/validate.py rename to tests/validate_test.py index c84ea5294..4ff44941f 100644 --- a/tests/validate.py +++ b/tests/validate_test.py @@ -2,15 +2,13 @@ import unittest from .helpers.ptrack_helpers import ProbackupTest, ProbackupException from datetime import datetime, timedelta +from pathlib import Path import subprocess from sys import exit import time import hashlib -module_name = 'validate' - - class ValidateTest(ProbackupTest, unittest.TestCase): # @unittest.skip("skip") @@ -19,12 +17,11 @@ def test_basic_validate_nullified_heap_page_backup(self): """ make node with nullified heap block """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -34,7 +31,7 @@ def test_basic_validate_nullified_heap_page_backup(self): file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('pgbench_accounts')").rstrip() + "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() node.safe_psql( "postgres", @@ -58,7 +55,7 @@ def test_basic_validate_nullified_heap_page_backup(self): with open(log_file_path) as f: log_content = f.read() self.assertIn( - 'File: "{0}" blknum 1, empty page'.format(file), + 'File: "{0}" blknum 1, empty page'.format(Path(file).as_posix()), log_content, 'Failed to detect nullified block') @@ -70,9 +67,6 @@ def test_basic_validate_nullified_heap_page_backup(self): pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_validate_wal_unreal_values(self): @@ -80,12 +74,11 @@ def test_validate_wal_unreal_values(self): make node with archiving, make archive backup validate to both real and unreal values """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -212,9 +205,6 @@ def test_validate_wal_unreal_values(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_basic_validate_corrupted_intermediate_backup(self): """ @@ -223,12 +213,11 @@ def test_basic_validate_corrupted_intermediate_backup(self): run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -244,7 +233,7 @@ def test_basic_validate_corrupted_intermediate_backup(self): "from generate_series(0,10000) i") file_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -297,9 +286,6 @@ def test_basic_validate_corrupted_intermediate_backup(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): """ @@ -308,12 +294,11 @@ def test_validate_corrupted_intermediate_backups(self): expect FULL and PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -326,7 +311,7 @@ def test_validate_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -337,7 +322,7 @@ def test_validate_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_path_t_heap_1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap_1')").rstrip() + "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -418,9 +403,6 @@ def test_validate_corrupted_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_error_intermediate_backups(self): """ @@ -430,12 +412,11 @@ def test_validate_specific_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -506,9 +487,6 @@ def test_validate_specific_error_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_error_intermediate_backups(self): """ @@ -518,12 +496,11 @@ def test_validate_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -590,9 +567,6 @@ def test_validate_error_intermediate_backups(self): self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups_1(self): """ @@ -601,12 +575,11 @@ def test_validate_corrupted_intermediate_backups_1(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -632,7 +605,7 @@ def test_validate_corrupted_intermediate_backups_1(self): "from generate_series(0,10000) i") file_page_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() backup_id_3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -662,7 +635,7 @@ def test_validate_corrupted_intermediate_backups_1(self): "from generate_series(0,10000) i") file_page_5 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() backup_id_6 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -787,9 +760,6 @@ def test_validate_corrupted_intermediate_backups_1(self): 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_target_corrupted_intermediate_backups(self): """ @@ -798,12 +768,11 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -829,7 +798,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_page_2 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() backup_id_3 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -853,7 +822,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "postgres", "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(30001, 30001) i RETURNING (xmin)").rstrip() + "from generate_series(30001, 30001) i RETURNING (xmin)").decode('utf-8').rstrip() backup_id_5 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -866,7 +835,7 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "from generate_series(0,10000) i") file_page_5 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() backup_id_6 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -979,8 +948,197 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups(self): + """ + make archive node, take FULL1, PAGE1_1, FULL2, PAGE2_1 backups, FULL3 + corrupt file in FULL and FULL2 and run validate on instance, + expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN + FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN + """ + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_2, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_4, + 'database', rel_path)) + + # Validate Instance + try: + self.validate_pb(backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"]) + self.assertEqual( + 1, 0, + "Expecting Error because of data files corruption.\n " + "Output: {0} \n CMD: {1}".format( + repr(self.output), self.cmd)) + except ProbackupException as e: + self.assertTrue( + "INFO: Validate backups of the instance 'node'" in e.message, + "\n Unexpected Error Message: {0}\n " + "CMD: {1}".format(repr(e.message), self.cmd)) + self.assertTrue( + 'WARNING: Some backups are not valid' in e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + # @unittest.skip("skip") + def test_validate_instance_with_several_corrupt_backups_interrupt(self): + """ + check that interrupt during validation is handled correctly + """ + self._check_gdb_flag_or_skip_test() + + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + initdb_params=['--data-checksums']) + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + self.set_archiving(backup_dir, 'node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select generate_series(0,1) i") + # FULL1 + backup_id_1 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # FULL2 + backup_id_2 = self.backup_node(backup_dir, 'node', node) + rel_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + node.safe_psql( + "postgres", + "insert into t_heap values(2)") + + backup_id_3 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL3 + backup_id_4 = self.backup_node(backup_dir, 'node', node) + + node.safe_psql( + "postgres", + "insert into t_heap values(3)") + + backup_id_5 = self.backup_node( + backup_dir, 'node', node, backup_type='page') + + # FULL4 + backup_id_6 = self.backup_node( + backup_dir, 'node', node, options=['--no-validate']) + + # Corrupt some files in FULL2 and FULL3 backup + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_1, + 'database', rel_path)) + os.remove(os.path.join( + backup_dir, 'backups', 'node', backup_id_3, + 'database', rel_path)) + + # Validate Instance + gdb = self.validate_pb( + backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) + + gdb.set_breakpoint('validate_file_pages') + gdb.run_until_break() + gdb.continue_execution_until_break() + gdb.remove_all_breakpoints() + gdb._execute('signal SIGINT') + gdb.continue_execution_until_error() + + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'Backup STATUS should be "CORRUPT"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'Backup STATUS should be "ORPHAN"') + self.assertEqual( + 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'Backup STATUS should be "OK"') + self.assertEqual( + 'DONE', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'Backup STATUS should be "OK"') + + log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + with open(log_file, 'r') as f: + log_content = f.read() + self.assertNotIn( + 'Interrupted while locking backup', log_content) # @unittest.skip("skip") def test_validate_instance_with_corrupted_page(self): @@ -989,12 +1147,11 @@ def test_validate_instance_with_corrupted_page(self): corrupt file in PAGE1 backup and run validate on instance, expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1015,7 +1172,7 @@ def test_validate_instance_with_corrupted_page(self): "from generate_series(0,10000) i") file_path_t_heap1 = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap1')").rstrip() + "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() # PAGE1 backup_id_2 = self.backup_node( backup_dir, 'node', node, backup_type='page') @@ -1126,20 +1283,16 @@ def test_validate_instance_with_corrupted_page(self): 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_corrupted_full_and_try_restore(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, try to restore backup with --no-validation option""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1152,7 +1305,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "from generate_series(0,10000) i") file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1222,19 +1375,15 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(self.output), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_instance_with_corrupted_full(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1248,7 +1397,7 @@ def test_validate_instance_with_corrupted_full(self): file_path_t_heap = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 backup_id_1 = self.backup_node(backup_dir, 'node', node) @@ -1316,18 +1465,14 @@ def test_validate_instance_with_corrupted_full(self): self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_1(self): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1378,17 +1523,13 @@ def test_validate_corrupt_wal_1(self): self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" - fname = self.id().split('.')[3] - node = self.make_simple_node(base_dir=os.path.join(module_name, fname, 'node'), + node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1444,9 +1585,6 @@ def test_validate_corrupt_wal_2(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_wal_lost_segment_1(self): """make archive node, make archive full backup, @@ -1454,12 +1592,11 @@ def test_validate_wal_lost_segment_1(self): run validate, expecting error because of missing wal segment make sure that backup status is 'CORRUPT' """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1521,21 +1658,17 @@ def test_validate_wal_lost_segment_1(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): """ make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1559,11 +1692,11 @@ def test_validate_corrupt_wal_between_backups(self): if self.get_version(node) < self.version_to_num('10.0'): walfile = node.safe_psql( 'postgres', - 'select pg_xlogfile_name(pg_current_xlog_location())').rstrip() + 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() else: walfile = node.safe_psql( 'postgres', - 'select pg_walfile_name(pg_current_wal_lsn())').rstrip() + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() if self.archive_compress: walfile = walfile + '.gz' @@ -1614,22 +1747,18 @@ def test_validate_corrupt_wal_between_backups(self): self.show_pb(backup_dir, 'node')[1]['status'], 'Backup STATUS should be "OK"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro702_688(self): """ make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) node.slow_start() @@ -1655,22 +1784,18 @@ def test_pgpro702_688(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_pgpro688(self): """ make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1684,9 +1809,6 @@ def test_pgpro688(self): backup_dir, 'node', options=["--time={0}".format(recovery_time), "-j", "4"]) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") # @unittest.expectedFailure def test_pgpro561(self): @@ -1694,13 +1816,12 @@ def test_pgpro561(self): make node with archiving, make stream backup, restore it to node1, check that archiving is not successful on node1 """ - fname = self.id().split('.')[3] node1 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node1'), + base_dir=os.path.join(self.module_name, self.fname, 'node1'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node1', node1) self.set_archiving(backup_dir, 'node1', node1) @@ -1710,7 +1831,7 @@ def test_pgpro561(self): backup_dir, 'node1', node1, options=["--stream"]) node2 = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node2')) + base_dir=os.path.join(self.module_name, self.fname, 'node2')) node2.cleanup() node1.psql( @@ -1792,9 +1913,6 @@ def test_pgpro561(self): self.assertFalse( 'pg_probackup archive-push completed successfully' in log_content) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full(self): """ @@ -1805,15 +1923,14 @@ def test_validate_corrupted_full(self): remove corruption and run valudate again, check that second full backup and his page backups are OK """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -1913,9 +2030,6 @@ def test_validate_corrupted_full(self): self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_1(self): """ @@ -1929,13 +2043,12 @@ def test_validate_corrupted_full_1(self): second page should be CORRUPT third page should be ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2015,9 +2128,6 @@ def test_validate_corrupted_full_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_2(self): """ @@ -2040,13 +2150,12 @@ def test_validate_corrupted_full_2(self): remove corruption from PAGE2_2 and run validate on PAGE2_4 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2377,9 +2486,6 @@ def test_validate_corrupted_full_2(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_corrupted_full_missing(self): """ @@ -2392,13 +2498,12 @@ def test_validate_corrupted_full_missing(self): second full backup and his firts page backups are OK, third page should be ORPHAN """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2610,18 +2715,14 @@ def test_validate_corrupted_full_missing(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - def test_file_size_corruption_no_validate(self): - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), # initdb_params=['--data-checksums'], ) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) @@ -2640,7 +2741,7 @@ def test_file_size_corruption_no_validate(self): heap_path = node.safe_psql( "postgres", - "select pg_relation_filepath('t_heap')").rstrip() + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() heap_size = node.safe_psql( "postgres", "select pg_relation_size('t_heap')") @@ -2672,9 +2773,6 @@ def test_file_size_corruption_no_validate(self): "ERROR: Backup files restoring failed" in e.message, repr(e.message)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup(self): """ @@ -2691,13 +2789,12 @@ def test_validate_specific_backup_with_missing_backup(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2815,9 +2912,6 @@ def test_validate_specific_backup_with_missing_backup(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup_1(self): """ @@ -2834,13 +2928,12 @@ def test_validate_specific_backup_with_missing_backup_1(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -2936,9 +3029,6 @@ def test_validate_specific_backup_with_missing_backup_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_with_missing_backup_1(self): """ @@ -2955,13 +3045,12 @@ def test_validate_with_missing_backup_1(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3125,9 +3214,6 @@ def test_validate_with_missing_backup_1(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validate_with_missing_backup_2(self): """ @@ -3144,13 +3230,12 @@ def test_validate_with_missing_backup_2(self): PAGE1_1 FULL1 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3285,19 +3370,15 @@ def test_validate_with_missing_backup_2(self): self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_corrupt_pg_control_via_resetxlog(self): """ PGPRO-2096 """ - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3355,16 +3436,14 @@ def test_corrupt_pg_control_via_resetxlog(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_validation_after_backup(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + self._check_gdb_flag_or_skip_test() + + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3393,19 +3472,15 @@ def test_validation_after_backup(self): self.show_pb(backup_dir, 'node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_corrupt_tablespace_map(self): """ Check that corruption in tablespace_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3448,19 +3523,16 @@ def test_validate_corrupt_tablespace_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - - # @unittest.expectedFailure + #TODO fix the test + @unittest.expectedFailure # @unittest.skip("skip") def test_validate_target_lsn(self): """ Check validation to specific LSN """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3479,7 +3551,7 @@ def test_validate_target_lsn(self): "from generate_series(0,10000) i") node_restored = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node_restored')) + base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) node_restored.cleanup() self.restore_node(backup_dir, 'node', node_restored) @@ -3505,17 +3577,13 @@ def test_validate_target_lsn(self): '--recovery-target-timeline=2', '--recovery-target-lsn={0}'.format(target_lsn)]) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_empty_and_mangled_database_map(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3579,16 +3647,12 @@ def test_partial_validate_empty_and_mangled_database_map(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_exclude(self): """""" - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3651,17 +3715,13 @@ def test_partial_validate_exclude(self): self.assertIn( "VERBOSE: Skip file validation due to partial restore", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - @unittest.skip("skip") def test_partial_validate_include(self): """ """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) self.init_pb(backup_dir) @@ -3713,18 +3773,14 @@ def test_partial_validate_include(self): self.assertNotIn( "VERBOSE: Skip file validation due to partial restore", output) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.skip("skip") def test_not_validate_diffenent_pg_version(self): """Do not validate backup, if binary is compiled with different PG version""" - fname = self.id().split('.')[3] node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), initdb_params=['--data-checksums']) - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') self.init_pb(backup_dir) self.add_instance(backup_dir, 'node', node) self.set_archiving(backup_dir, 'node', node) @@ -3746,7 +3802,9 @@ def test_not_validate_diffenent_pg_version(self): with open(control_file, 'r') as f: data = f.read(); - data = data.replace(str(pg_version), str(fake_new_pg_version)) + data = data.replace( + "server-version = {0}".format(str(pg_version)), + "server-version = {0}".format(str(fake_new_pg_version))) with open(control_file, 'w') as f: f.write(data); @@ -3765,19 +3823,15 @@ def test_not_validate_diffenent_pg_version(self): "\n Unexpected Error Message: {0}\n CMD: {1}".format( repr(e.message), self.cmd)) - # Clean after yourself - self.del_test_dir(module_name, fname) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_corrupt_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3801,46 +3855,34 @@ def test_validate_corrupt_page_header_map(self): f.seek(42) f.write(b"blah") f.flush() - f.close - try: + with self.assertRaises(ProbackupException) as cm: self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + e = cm.exception + self.assertRegex( + cm.exception.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - try: + self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + + with self.assertRaises(ProbackupException) as cm: self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: An error occured during metadata decompression' in e.message and - 'data error' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + e = cm.exception + self.assertRegex( + e.message, + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) - self.assertIn("WARNING: Some backups are not valid", e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) + self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) + self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + self.assertIn("WARNING: Some backups are not valid", e.message) # @unittest.expectedFailure # @unittest.skip("skip") @@ -3848,10 +3890,9 @@ def test_validate_truncated_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3902,19 +3943,15 @@ def test_validate_truncated_page_header_map(self): self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) self.assertIn("WARNING: Some backups are not valid", e.message) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) - # @unittest.expectedFailure # @unittest.skip("skip") def test_validate_missing_page_header_map(self): """ Check that corruption in page_header_map is detected """ - fname = self.id().split('.')[3] - backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup') + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') node = self.make_simple_node( - base_dir=os.path.join(module_name, fname, 'node'), + base_dir=os.path.join(self.module_name, self.fname, 'node'), set_replication=True, initdb_params=['--data-checksums']) @@ -3962,8 +3999,66 @@ def test_validate_missing_page_header_map(self): self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) self.assertIn("WARNING: Some backups are not valid", e.message) - # Clean after yourself - self.del_test_dir(module_name, fname, [node]) + # @unittest.expectedFailure + # @unittest.skip("skip") + def test_no_validate_tablespace_map(self): + """ + Check that --no-validate is propagated to tablespace_map + """ + backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.make_simple_node( + base_dir=os.path.join(self.module_name, self.fname, 'node'), + set_replication=True, + initdb_params=['--data-checksums']) + + self.init_pb(backup_dir) + self.add_instance(backup_dir, 'node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'external_dir') + + node.safe_psql( + 'postgres', + 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') + + tblspace_new = self.get_tblspace_path(node, 'external_dir_new') + + oid = node.safe_psql( + 'postgres', + "select oid from pg_tablespace where spcname = 'external_dir'").decode('utf-8').rstrip() + + # FULL backup + backup_id = self.backup_node( + backup_dir, 'node', node, options=['--stream']) + + pgdata = self.pgdata_content(node.data_dir) + + tablespace_map = os.path.join( + backup_dir, 'backups', 'node', + backup_id, 'database', 'tablespace_map') + + # overwrite tablespace_map file + with open(tablespace_map, "w") as f: + f.write("{0} {1}".format(oid, tblspace_new)) + f.close + + node.cleanup() + + self.restore_node(backup_dir, 'node', node, options=['--no-validate']) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # check that tablespace restore as symlink + tablespace_link = os.path.join(node.data_dir, 'pg_tblspc', oid) + self.assertTrue( + os.path.islink(tablespace_link), + "'%s' is not a symlink" % tablespace_link) + + self.assertEqual( + os.readlink(tablespace_link), + tblspace_new, + "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) # validate empty backup list # page from future during validate @@ -3976,4 +4071,4 @@ def test_validate_missing_page_header_map(self): # 715 MAXALIGN(header.compressed_size), in); # 716 if (read_len != MAXALIGN(header.compressed_size)) # -> 717 elog(ERROR, "cannot read block %u of \"%s\" read %lu of %d", -# 718 blknum, file->path, read_len, header.compressed_size); \ No newline at end of file +# 718 blknum, file->path, read_len, header.compressed_size); diff --git a/travis/Dockerfile.in b/travis/Dockerfile.in deleted file mode 100644 index a3c858ee2..000000000 --- a/travis/Dockerfile.in +++ /dev/null @@ -1,28 +0,0 @@ -FROM ololobus/postgres-dev:stretch - -USER root -RUN apt-get update -RUN apt-get -yq install python python-pip - -# RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -# RUN python2 get-pip.py -RUN python2 -m pip install virtualenv - -# Environment -ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH} -ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin - -# Make directories -RUN mkdir -p /pg/testdir - -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -COPY . /pg/testdir -WORKDIR /pg/testdir - -# Grant privileges -RUN chown -R postgres:postgres /pg/testdir - -USER postgres -ENTRYPOINT MODE=${MODE} /run.sh diff --git a/travis/backup_restore.sh b/travis/backup_restore.sh deleted file mode 100644 index b3c9df1ed..000000000 --- a/travis/backup_restore.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -ex - -# vars -export PGVERSION=9.5.4 -export PATH=$PATH:/usr/pgsql-9.5/bin -export PGUSER=pgbench -export PGDATABASE=pgbench -export PGDATA=/var/lib/pgsql/9.5/data -export BACKUP_PATH=/backups -export ARCLOG_PATH=$BACKUP_PATH/backup/pg_xlog -export PGDATA2=/var/lib/pgsql/9.5/data2 -export PGBENCH_SCALE=100 -export PGBENCH_TIME=60 - -# prepare directory -cp -a /tests /build -pushd /build - -# download postgresql -yum install -y wget -wget -k https://ftp.postgresql.org/pub/source/v$PGVERSION/postgresql-$PGVERSION.tar.gz -O postgresql.tar.gz -tar xf postgresql.tar.gz - -# install pg_probackup -yum install -y https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-2.noarch.rpm -yum install -y postgresql95-devel make gcc readline-devel openssl-devel pam-devel libxml2-devel libxslt-devel -make top_srcdir=postgresql-$PGVERSION -make install top_srcdir=postgresql-$PGVERSION - -# initialize cluster and database -yum install -y postgresql95-server -su postgres -c "/usr/pgsql-9.5/bin/initdb -D $PGDATA -k" -cat < $PGDATA/pg_hba.conf -local all all trust -host all all 127.0.0.1/32 trust -local replication pgbench trust -host replication pgbench 127.0.0.1/32 trust -EOF -cat < $PGDATA/postgresql.auto.conf -max_wal_senders = 2 -wal_level = logical -wal_log_hints = on -EOF -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA" -su postgres -c "createdb -U postgres $PGUSER" -su postgres -c "createuser -U postgres -a -d -E $PGUSER" -pgbench -i -s $PGBENCH_SCALE - -# Count current -COUNT=$(psql -Atc "select count(*) from pgbench_accounts") -pgbench -s $PGBENCH_SCALE -T $PGBENCH_TIME -j 2 -c 10 & - -# create backup -pg_probackup init -pg_probackup backup -b full --disable-ptrack-clear --stream -v -pg_probackup show -sleep $PGBENCH_TIME - -# restore from backup -chown -R postgres:postgres $BACKUP_PATH -su postgres -c "pg_probackup restore -D $PGDATA2" - -# start backup server -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl stop -w -D $PGDATA" -su postgres -c "/usr/pgsql-9.5/bin/pg_ctl start -w -D $PGDATA2" -( psql -Atc "select count(*) from pgbench_accounts" | grep $COUNT ) || (cat $PGDATA2/pg_log/*.log ; exit 1) diff --git a/travis/before-install.sh b/travis/before-install.sh new file mode 100755 index 000000000..376de5e6e --- /dev/null +++ b/travis/before-install.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -xe + +mkdir /pg +chown travis /pg \ No newline at end of file diff --git a/travis/before-script-user.sh b/travis/before-script-user.sh new file mode 100755 index 000000000..d9c07f1e4 --- /dev/null +++ b/travis/before-script-user.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -xe + +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N "" +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +ssh-keyscan -H localhost >> ~/.ssh/known_hosts diff --git a/travis/before-script.sh b/travis/before-script.sh new file mode 100755 index 000000000..ca59bcf23 --- /dev/null +++ b/travis/before-script.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -xe + +/etc/init.d/ssh start + +# Show pg_config path (just in case) +echo "############### pg_config path:" +which pg_config + +# Show pg_config just in case +echo "############### pg_config:" +pg_config + +# Show kernel parameters +echo "############### kernel params:" +cat /proc/sys/kernel/yama/ptrace_scope +sudo sysctl kernel.yama.ptrace_scope=0 +cat /proc/sys/kernel/yama/ptrace_scope diff --git a/travis/docker-compose.yml b/travis/docker-compose.yml deleted file mode 100644 index 471ab779f..000000000 --- a/travis/docker-compose.yml +++ /dev/null @@ -1,2 +0,0 @@ -tests: - build: . diff --git a/travis/install.sh b/travis/install.sh new file mode 100755 index 000000000..43ada47b7 --- /dev/null +++ b/travis/install.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -xe + +if [ -z ${PG_VERSION+x} ]; then + echo PG_VERSION is not set! + exit 1 +fi + +if [ -z ${PG_BRANCH+x} ]; then + echo PG_BRANCH is not set! + exit 1 +fi + +if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then + PTRACK_PATCH_PG_BRANCH=OFF +fi + +# fix +sudo chown -R travis /home/travis/.ccache + +export PGHOME=/pg + +# Clone Postgres +echo "############### Getting Postgres sources:" +git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 + +# Clone ptrack +if [ "$PTRACK_PATCH_PG_BRANCH" != "OFF" ]; then + git clone https://github.com/postgrespro/ptrack.git -b master --depth=1 postgres/contrib/ptrack + export PG_PROBACKUP_PTRACK=ON +else + export PG_PROBACKUP_PTRACK=OFF +fi + +# Compile and install Postgres +echo "############### Compiling Postgres:" +cd postgres # Go to postgres dir +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then + git apply -3 contrib/ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff +fi +CC='ccache gcc' CFLAGS="-Og" ./configure --prefix=$PGHOME \ + --cache-file=~/.ccache/configure-cache \ + --enable-debug --enable-cassert --enable-depend \ + --enable-tap-tests --enable-nls +make -s -j$(nproc) install +make -s -j$(nproc) -C contrib/ install + +# Override default Postgres instance +export PATH=$PGHOME/bin:$PATH +export LD_LIBRARY_PATH=$PGHOME/lib +export PG_CONFIG=$(which pg_config) + +if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then + echo "############### Compiling Ptrack:" + make -C contrib/ptrack install +fi + +# Get amcheck if missing +if [ ! -d "contrib/amcheck" ]; then + echo "############### Getting missing amcheck:" + git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck + make -C contrib/amcheck install +fi + +pip3 install testgres \ No newline at end of file diff --git a/travis/make_dockerfile.sh b/travis/make_dockerfile.sh deleted file mode 100755 index 3e6938bd9..000000000 --- a/travis/make_dockerfile.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env sh - -if [ -z ${PG_VERSION+x} ]; then - echo PG_VERSION is not set! - exit 1 -fi - -if [ -z ${PG_BRANCH+x} ]; then - echo PG_BRANCH is not set! - exit 1 -fi - -if [ -z ${MODE+x} ]; then - MODE=basic -fi - -echo PG_VERSION=${PG_VERSION} -echo PG_BRANCH=${PG_BRANCH} -echo MODE=${MODE} - -sed \ - -e 's/${PG_VERSION}/'${PG_VERSION}/g \ - -e 's/${PG_BRANCH}/'${PG_BRANCH}/g \ - -e 's/${MODE}/'${MODE}/g \ -Dockerfile.in > Dockerfile diff --git a/travis/run_tests.sh b/travis/run_tests.sh deleted file mode 100755 index 1bb3a6fde..000000000 --- a/travis/run_tests.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2019-2020, Postgres Professional -# - - -PG_SRC=$PWD/postgres - -# # Here PG_VERSION is provided by postgres:X-alpine docker image -# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2 -# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c - - -# mkdir $PG_SRC - -# tar \ -# --extract \ -# --file postgresql.tar.bz2 \ -# --directory $PG_SRC \ -# --strip-components 1 - -# Clone Postgres -echo "############### Getting Postgres sources:" -git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 - -# Compile and install Postgres -echo "############### Compiling Postgres:" -cd postgres # Go to postgres dir -./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests -make -s -j$(nproc) install -make -s -j$(nproc) -C contrib/ install - -# Override default Postgres instance -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib -export PG_CONFIG=$(which pg_config) - -# Get amcheck if missing -if [ ! -d "contrib/amcheck" ]; then - echo "############### Getting missing amcheck:" - git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck - make USE_PGXS=1 -C contrib/amcheck install -fi - -# Get back to testdir -cd .. - -# Show pg_config path (just in case) -echo "############### pg_config path:" -which pg_config - -# Show pg_config just in case -echo "############### pg_config:" -pg_config - -# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) -echo "############### Compiling and installing pg_probackup:" -# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install -make USE_PGXS=1 top_srcdir=$PG_SRC install - -# Setup python environment -echo "############### Setting up python env:" -python2 -m virtualenv pyenv -source pyenv/bin/activate -pip install testgres==1.8.2 - -echo "############### Testing:" -if [ "$MODE" = "basic" ]; then - export PG_PROBACKUP_TEST_BASIC=ON - python -m unittest -v tests - python -m unittest -v tests.init -else - python -m unittest -v tests.$MODE -fi - -# Generate *.gcov files -# gcov src/*.c src/*.h - -# Send coverage stats to Codecov -# bash <(curl -s https://codecov.io/bash) diff --git a/travis/script.sh b/travis/script.sh new file mode 100755 index 000000000..31ef09726 --- /dev/null +++ b/travis/script.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -xe + +export PGHOME=/pg +export PG_SRC=$PWD/postgres +export PATH=$PGHOME/bin:$PATH +export LD_LIBRARY_PATH=$PGHOME/lib +export PG_CONFIG=$(which pg_config) + +# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) +echo "############### Compiling and installing pg_probackup:" +# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install +make USE_PGXS=1 top_srcdir=$PG_SRC install + +if [ -z ${MODE+x} ]; then + MODE=basic +fi + +if [ -z ${PGPROBACKUP_GDB+x} ]; then + PGPROBACKUP_GDB=ON +fi + +echo "############### Testing:" +echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA} +echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION} +echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD} +echo PGPROBACKUPBIN=${PGPROBACKUPBIN} +echo PGPROBACKUP_SSH_REMOTE=${PGPROBACKUP_SSH_REMOTE} +echo PGPROBACKUP_GDB=${PGPROBACKUP_GDB} +echo PG_PROBACKUP_PTRACK=${PG_PROBACKUP_PTRACK} + +if [ "$MODE" = "basic" ]; then + export PG_PROBACKUP_TEST_BASIC=ON + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} + python3 -m unittest -v tests + python3 -m unittest -v tests.init_test +else + echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} + python3 -m unittest -v tests.$MODE +fi