Merge pull request vitabaks#80 from vitabaks/dev

custom WAL location (Issue vitabaks#75)

Author: vitabaks

roles/patroni/tasks/custom_wal_dir.yml

@@ -0,0 +1,91 @@
+---
+# yamllint disable rule:line-length
+# yamllint disable rule:comments-indentation
+
+- name: "Make sure {{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }} is symlink"  # noqa 204
+  stat:
+    path: "{{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}"
+  register: sym
+
+- block:  # synchronize WAL`s if wal dir exist (and is not symlink)
+    - name: Make sure rsync is installed (for synchronize wal dir)
+      package:
+        name:
+          - rsync
+          - sshpass
+        state: present
+      environment: "{{ proxy_env | default({}) }}"
+
+    - name: Stop patroni service (for create symlink)
+      systemd:
+        name: patroni
+        state: stopped
+      register: patroni_stop_result
+
+    - name: Make sure PostgreSQL is stopped
+      become: true
+      become_user: postgres
+      command: "{{ postgresql_bin_dir }}/pg_ctl status -D {{ postgresql_data_dir }}"
+      register: stop_result
+      changed_when: false
+      failed_when: false
+      until: stop_result.rc == 3
+      retries: 100
+      delay: 6
+
+    - name: "Synchronize {{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }} to {{ postgresql_wal_dir }}"  # noqa 204
+      become: true
+      become_user: postgres
+      synchronize:
+        src: "{{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}/"
+        dest: "{{ postgresql_wal_dir }}/"
+      delegate_to: "{{ inventory_hostname }}"
+
+    - name: "Rename {{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }} to {{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}_old"  # noqa 204
+      command: mv {{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }} {{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}_old  # noqa 204
+      register: mv_result
+  when: sym.stat.exists and not sym.stat.islnk|bool
+
+- name: "Create symlink {{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }} -> {{ postgresql_wal_dir }}"  # noqa 204
+  become: true
+  become_user: postgres
+  file:
+    src: "{{ postgresql_wal_dir }}"
+    dest: "{{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}"
+    state: link
+  when: sym.stat.islnk is not defined or not sym.stat.islnk|bool
+
+- block:  # start patroni
+    - name: Start patroni service
+      systemd:
+        name: patroni
+        state: started
+
+    - name: Wait for port 8008 to become open on the host
+      wait_for:
+        port: 8008
+        host: "{{ hostvars[inventory_hostname]['inventory_hostname'] }}"
+        state: started
+        timeout: 120
+        delay: 10
+      ignore_errors: false
+
+    - name: Check that the patroni is healthy
+      uri:
+        url: "http://{{ hostvars[inventory_hostname]['inventory_hostname'] }}:8008/health"
+        status_code: 200
+      register: replica_result
+      until: replica_result.status == 200
+      retries: 120
+      delay: 10
+  when: patroni_stop_result is changed
+
+- name: "Remove {{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}_old directory"
+  file:
+    path: "{{ postgresql_data_dir }}/{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}_old"
+    state: absent
+  when:
+    - replica_result.status is defined
+    - replica_result.status == 200
+
+...

roles/patroni/tasks/main.yml

@@ -342,6 +342,16 @@
     mode: 0700
   tags: patroni
 
+- name: Prepare PostgreSQL | make sure the custom WAL directory "{{ postgresql_wal_dir }}" exists
+  file:
+    path: "{{ postgresql_wal_dir }}"
+    owner: postgres
+    group: postgres
+    state: directory
+    mode: 0700
+  when: postgresql_wal_dir is defined and postgresql_wal_dir | length > 0
+  tags: patroni, custom_wal_dir
+
 - block:  # when postgresql exists (master)
     - name: Prepare PostgreSQL | check that data directory "{{ postgresql_data_dir }}" is initialized on Master
       stat:
@@ -521,7 +531,7 @@
             postgresql_packages|join(" ") is not search("postgrespro")) and
             not postgresql_conf_file.stat.exists
 
-    - name: Prepare PostgreSQL | make sure the data directory "{{ postgresql_data_dir }}" is empty on Master
+    - name: Prepare PostgreSQL | make sure the data directory "{{ postgresql_data_dir }}" is empty
       file:
         path: "{{ postgresql_data_dir }}"
         state: "{{ item }}"
@@ -531,7 +541,19 @@
       loop:
         - absent
         - directory
-      when: is_master == "true" and
+      when: patroni_cluster_bootstrap_method != "pgbackrest"  # --delta restore
+
+    - name: Prepare PostgreSQL | make sure the custom WAL directory "{{ postgresql_wal_dir }}" is empty
+      file:
+        path: "{{ postgresql_wal_dir }}"
+        state: "{{ item }}"
+        owner: postgres
+        group: postgres
+        mode: 0700
+      loop:
+        - absent
+        - directory
+      when: (postgresql_wal_dir is defined and postgresql_wal_dir | length > 0) and
             patroni_cluster_bootstrap_method != "pgbackrest"  # --delta restore
   when: postgresql_exists != "true"
   tags: patroni, point_in_time_recovery
@@ -710,6 +732,7 @@
         group: postgres
         mode: 0640
   when: patroni_cluster_bootstrap_method != "initdb" and
+        postgresql_conf_dir != postgresql_data_dir and
         (existing_pgcluster is not defined or not existing_pgcluster|bool)
   tags: patroni
 
@@ -752,13 +775,24 @@
   tags: patroni, patroni_start_master, point_in_time_recovery
 
 - block:  # pg_hba (using a templates/pg_hba.conf.j2)
-    - name: Prepare PostgreSQL | generate pg_hba.conf
+    - name: Prepare PostgreSQL | generate pg_hba.conf on Master
       template:
         src: templates/pg_hba.conf.j2
         dest: "{{ postgresql_conf_dir }}/pg_hba.conf"
         owner: postgres
         group: postgres
         mode: 0640
+      when: is_master == "true"
+
+    - name: Prepare PostgreSQL | generate pg_hba.conf on Replica
+      template:
+        src: templates/pg_hba.conf.j2
+        dest: "{{ postgresql_conf_dir }}/pg_hba.conf"
+        owner: postgres
+        group: postgres
+        mode: 0640
+      when: is_master != "true" and
+            postgresql_conf_dir != postgresql_data_dir
 
     - name: Prepare PostgreSQL | reload for apply the pg_hba.conf
       become: true
@@ -837,21 +871,10 @@
         group: postgres
         mode: 0640
   when: existing_pgcluster is defined and existing_pgcluster|bool
+        and postgresql_conf_dir != postgresql_data_dir
   tags: patroni, pg_hba, pg_hba_generate
 
 - block:  # start patroni on replica
-    - name: Prepare PostgreSQL | make sure the data directory "{{ postgresql_data_dir }}" is empty on Replica
-      file:
-        path: "{{ postgresql_data_dir }}"
-        state: "{{ item }}"
-        owner: postgres
-        group: postgres
-        mode: 0700
-      loop:
-        - absent
-        - directory
-      when: not 'pgbackrest' in patroni_create_replica_methods  # --delta restore
-
     - name: Start patroni service on Replica servers
       systemd:
         daemon_reload: true
@@ -879,6 +902,11 @@
   when: is_master != "true"
   tags: patroni, patroni_start_replica, point_in_time_recovery
 
+# create symlink pg_xlog/pg_wal to custom WAL dir
+- import_tasks: custom_wal_dir.yml
+  when: postgresql_wal_dir is defined and postgresql_wal_dir | length > 0
+  tags: patroni, custom_wal_dir
+
 # disable postgresql from autostart
 - block:  # "Debian"
     - name: Turning off postgresql autostart from config "start.conf" (will be managed by patroni)

roles/patroni/templates/patroni.yml.j2

@@ -148,6 +148,9 @@ postgresql:
     {% for item in basebackup %}
     {{ item.option }}: '{{ item.value }}'
     {% endfor %}
+    {% if postgresql_version is version('10', '>=') and postgresql_wal_dir is defined and postgresql_wal_dir | length > 0 %}
+    waldir: {{ postgresql_wal_dir }}
+    {% endif %}
   {% endif %}
 {% else %}
     - basebackup

tags.md

@@ -38,6 +38,7 @@
 - - patroni_start_master
 - - patroni_start_replica
 - - postgresql_disable
+- - custom_wal_dir
 - pgbouncer
 - - pgbouncer_install
 - - pgbouncer_conf

vars/Debian.yml

@@ -3,7 +3,8 @@
 
 # PostgreSQL variables
 postgresql_cluster_name: "main"
-postgresql_data_dir: "/var/lib/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}"
+postgresql_data_dir: "/var/lib/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}"  # You can specify custom data dir path
+postgresql_wal_dir: ""  # custom WAL dir path (symlink will be created) [optional]
 postgresql_conf_dir: "/etc/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}"
 postgresql_bin_dir: "/usr/lib/postgresql/{{ postgresql_version }}/bin"
 postgresql_log_dir: "/var/log/postgresql"

vars/RedHat.yml

@@ -2,7 +2,8 @@
 # yamllint disable rule:line-length
 
 # PostgreSQL variables
-postgresql_data_dir: "/var/lib/pgsql/{{ postgresql_version }}/data"
+postgresql_data_dir: "/var/lib/pgsql/{{ postgresql_version }}/data"  # You can specify custom data dir path
+postgresql_wal_dir: ""  # custom WAL dir path (symlink will be created) [optional]
 postgresql_conf_dir: "{{ postgresql_data_dir }}"
 postgresql_bin_dir: "/usr/pgsql-{{ postgresql_version }}/bin"
 postgresql_log_dir: "/var/log/postgresql"