From 168be1b75da0e9f90d72c3eed678123d9b9909de Mon Sep 17 00:00:00 2001 From: dragarcia Date: Tue, 17 Aug 2021 18:25:18 +0800 Subject: [PATCH 01/29] chore: bump postgres and pgbouncer - #67 #68 --- ansible/vars.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 614f595c6..7d15eb3bb 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,12 +1,12 @@ supabase_internal: true postgresql_major: "13" -postgresql_release: "13.3" -postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa +postgresql_release: "13.4" +postgresql_release_checksum: sha1:100ab62f9ef5dbd90f83c5da284e24ab0070ddb4 # Non Postgres Extensions -pgbouncer_release: "1.15.0" -pgbouncer_release_checksum: sha1:ea7e9dbcab178f439a0fa402a78a7f1e4f43e6d4 +pgbouncer_release: "1.16.0" +pgbouncer_release_checksum: sha1:7d4b7e1110387df2245b22de3168884fdc8092c4 postgrest_arm_release: 2021-03-05-19-03-d3a8b5f-ubuntu-aarch64 postgrest_arm_release_checksum: sha1:b9e9b06ead7230b75033e8ae17912714bf463a33 From e1611f690ab03a85a63b662f63115c396848f6cd Mon Sep 17 00:00:00 2001 From: dragarcia Date: Tue, 17 Aug 2021 18:26:09 +0800 Subject: [PATCH 02/29] chore: #65 sha-scram-256 encryption --- ansible/files/pgbouncer_config/pgbouncer.ini.j2 | 2 +- ansible/files/postgresql_config/pg_hba.conf.j2 | 4 ++-- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 index c0c5dd0f6..6ebce5ca6 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -113,7 +113,7 @@ unix_socket_dir = /tmp ;;; ;; any, trust, plain, md5, cert, hba, pam -auth_type = md5 +auth_type = scram-sha-256 auth_file = /etc/pgbouncer/userlist.txt ;; Path to HBA-style auth config diff --git a/ansible/files/postgresql_config/pg_hba.conf.j2 b/ansible/files/postgresql_config/pg_hba.conf.j2 index 1e4c86604..17b8d7c15 100755 --- a/ansible/files/postgresql_config/pg_hba.conf.j2 +++ b/ansible/files/postgresql_config/pg_hba.conf.j2 @@ -84,11 +84,11 @@ local all all peer # IPv4 local connections: host all all 127.0.0.1/32 trust # IPv6 local connections: -host all all ::1/128 md5 +host all all ::1/128 scram-sha-256 # Local root Unix user, passwordless access local all postgres peer map=root_as_postgres # IPv4 external connections -host all all 0.0.0.0/0 md5 +host all all 0.0.0.0/0 scram-sha-256 # MD5 hashed password hosts diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 8b4fec8e2..e15022dbb 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -92,7 +92,7 @@ listen_addresses = '*' # what IP address(es) to listen on; # - Authentication - authentication_timeout = 1min # 1s-600s -password_encryption = md5 # scram-sha-256 or md5 +password_encryption = scram-sha-256 # scram-sha-256 or md5 db_user_namespace = off # GSSAPI using Kerberos From 9461f362b20baa56fac35de9f908c5efb0815cb7 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Wed, 10 Nov 2021 15:08:50 +0800 Subject: [PATCH 03/29] chore: Bump to Postgres 14.0 --- ansible/vars.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 4f48a3e97..2a4ebed30 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,8 +1,8 @@ supabase_internal: true -postgresql_major: "13" -postgresql_release: "13.4" -postgresql_release_checksum: sha1:100ab62f9ef5dbd90f83c5da284e24ab0070ddb4 +postgresql_major: "14" +postgresql_release: "14.0" +postgresql_release_checksum: sha1:a58a5492b25d89c79e7a3727e09dd2847b2b3f33 # Non Postgres Extensions pgbouncer_release: "1.16.0" From d8fae0f7593c193a8f701cfbdd48058b1c221f52 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Wed, 10 Nov 2021 15:25:59 +0800 Subject: [PATCH 04/29] chore: bump up existing extensions --- ansible/vars.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 2a4ebed30..d76034f0d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -39,47 +39,47 @@ adminapi_release_checksum: amd64: sha256:a15eb38d633aaf8d227c1d1a82cdf1359b85624a96f80cf826c664d779767dd7 # Postgres Extensions -postgis_release: "3.1.2" -postgis_release_checksum: sha1:622f52f3bf338c8e51ea6d73d30d6a5d3140c517 +postgis_release: "3.1.4" +postgis_release_checksum: sha1:3077da5136841d9d51a4325a233be8eccf763c38 -pgrouting_release: "3.2.0" -pgrouting_release_checksum: sha1:d902d449ebc96b6cdcb2fac09434d0098467cda5 +pgrouting_release: "3.2.2" +pgrouting_release_checksum: sha1:56596fa6e22104572d61296e1fa25e7918c79671 pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e -pg_cron_release: "1.3.1" -pg_cron_release_checksum: sha1:679b6ff54e0b1070a5fd713c5d25c3378f371fac +pg_cron_release: "1.4.1" +pg_cron_release_checksum: sha1:411cb25b64a3aefc26e9b0c9d9de205cbeefe631 -pgaudit_release: "1.5.0" -pgaudit_release_checksum: sha1:8429125e8f70fcaa2c2f5a0e22b910a4afb821a4 +pgaudit_release: "1.6.1" +pgaudit_release_checksum: sha1:2fae2690d38a1822b6e8f40ad6b2bd8669498659 pgsql_http_release: "1.3.1" pgsql_http_release_checksum: sha1:816a3fff53e05301b176cf0696799fc5a00f54e8 -plpgsql_check_release: "1.16.0" -plpgsql_check_release_checksum: sha1:626553fc2746fe10aa5a776a1229baf2af3365fc +plpgsql_check_release: "2.0.5" +plpgsql_check_release_checksum: sha1:7fc9181f291bb0b24a7886681ab8bb837041ab62 pg_safeupdate_release: "1.3" pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c timescaledb_release: "2.3.0" -wal2json_release: "2_3" -wal2json_release_checksum: sha1:923f9bbcd0505a1f0b6eac1d371e4ff2d266a958 +wal2json_release: "2_4" +wal2json_release_checksum: sha1:9f5c8dec3c0a5c19b1b77273f67fd9b0b0ab6664 supautils_release: "1.1.1" supautils_release_checksum: sha1:431e1be36011026c7765dc365fa36cd5d6162f11 -pljava_release: "1_6_2" -pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 +pljava_release: "1_6_3" +pljava_release_checksum: sha1:550bea791c404c9d62050fd9c330e162bab20763 plv8_commit_version: 3656177d384e3e02b74faa8e2931600f3690ab59 pg_plan_filter_commit_version: 5081a7b5cb890876e67d8e7486b6a64c38c9a492 -pg_net_release: "0.2" -pg_net_release_checksum: sha1:22c40ae9039778a6bf7344e4357640edf8620a7e +pg_net_release: "0.3" +pg_net_release_checksum: sha1:0695ad2e4e9b2a35a77dfa89bd4c5a90c622eb24 vector_x86_deb: 'https://packages.timber.io/vector/0.17.0/vector-0.17.0-amd64.deb' vector_arm_deb: 'https://packages.timber.io/vector/0.17.0/vector-0.17.0-arm64.deb' From 931e55b555a11d3de58e39ceb9a34a55440ef24f Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 15 Nov 2021 14:42:18 +0800 Subject: [PATCH 05/29] Use PG 14 version of postgresql.conf --- .../postgresql_config/postgresql.conf.j2 | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 425f10729..ddd94cfec 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -24,7 +24,8 @@ # "postgres -c log_connections=on". Some parameters can be changed at run time # with the "SET" SQL command. # -# Memory units: kB = kilobytes Time units: ms = milliseconds +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds # MB = megabytes s = seconds # GB = gigabytes min = minutes # TB = terabytes h = hours @@ -105,6 +106,7 @@ ssl = off ssl_ca_file = '' ssl_cert_file = '' ssl_crl_file = '' +ssl_crl_dir = '' ssl_key_file = '' ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers ssl_prefer_server_ciphers = on @@ -124,28 +126,28 @@ ssl_passphrase_command_supports_reload = off shared_buffers = 128MB # min 128kB # (change requires restart) -# huge_pages = try # on, off, or try +#huge_pages = try # on, off, or try # (change requires restart) -# huge_page_size = 0 # zero for system default +#huge_page_size = 0 # zero for system default # (change requires restart) -# temp_buffers = 8MB # min 800kB -# max_prepared_transactions = 0 # zero disables the feature +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. -# work_mem = 4MB # min 64kB -# hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem -# maintenance_work_mem = 64MB # min 1MB -# autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem -# logical_decoding_work_mem = 64MB # min 64kB -# max_stack_depth = 2MB # min 100kB -# shared_memory_type = mmap # the default is the first option +#work_mem = 4MB # min 64kB +#hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#logical_decoding_work_mem = 64MB # min 64kB +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option # supported by the operating system: # mmap # sysv # windows # (change requires restart) -# dynamic_shared_memory_type = posix # the default is the first option +#dynamic_shared_memory_type = posix # the default is the first option # supported by the operating system: # posix # sysv @@ -765,8 +767,6 @@ jit_provider = 'llvmjit' # JIT library to use #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? -#remove_temp_files_after_crash = on # remove temporary files after - # backend crash? #data_sync_retry = off # retry or panic on failure to fsync # data? # (change requires restart) From 6e04163435592432e56b737f018fd0ac9582fd03 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:08:12 +0800 Subject: [PATCH 06/29] move up running of SQL files - to a part where the DB is still up and running --- ansible/playbook.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 833bfa49c..319cd8e2c 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -26,6 +26,25 @@ name: postgresql state: started + - name: Transfer init SQL files + copy: + src: files/{{ item.source }} + dest: /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Execute init SQL files + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Delete SQL scripts + file: + path: /tmp/{{ item.dest }} + state: absent + loop: "{{ sql_files }}" + - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml @@ -52,25 +71,6 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - - name: Transfer init SQL files - copy: - src: files/{{ item.source }} - dest: /tmp/{{ item.dest }} - loop: "{{ sql_files }}" - - - name: Execute init SQL files - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} - loop: "{{ sql_files }}" - - - name: Delete SQL scripts - file: - path: /tmp/{{ item.dest }} - state: absent - loop: "{{ sql_files }}" - - name: UFW - Allow SSH connections ufw: rule: allow From 9bc3cd6e05f9af55c4d47f92091d185a94aebde8 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:08:39 +0800 Subject: [PATCH 07/29] new extensions: - rum - pg_hashids --- ansible/tasks/postgres-extensions/16-rum.yml | 28 +++++++++++++++++++ .../postgres-extensions/17-pg_hashids.yml | 17 +++++++++++ ansible/tasks/setup-extensions.yml | 8 +++++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 ansible/tasks/postgres-extensions/16-rum.yml create mode 100644 ansible/tasks/postgres-extensions/17-pg_hashids.yml diff --git a/ansible/tasks/postgres-extensions/16-rum.yml b/ansible/tasks/postgres-extensions/16-rum.yml new file mode 100644 index 000000000..9a2fad7ac --- /dev/null +++ b/ansible/tasks/postgres-extensions/16-rum.yml @@ -0,0 +1,28 @@ +# rum +- name: rum - download latest release + get_url: + url: "https://github.com/postgrespro/rum/archive/refs/tags/%7B%7Brum_release%7D%7D.tar.gz" + dest: /tmp/rum-{{ rum_release }}.tar.gz + checksum: "{{ rum_release_checksum }}" + +- name: rum - unpack archive + unarchive: + remote_src: yes + src: /tmp/rum-{{ rum_release }}.tar.gz + dest: /tmp + become: yes + +- name: rum - build + make: + chdir: /tmp/rum-{{ rum_release }} + params: + USE_PGXS: 1 + become: yes + +- name: rum - install + make: + chdir: /tmp/rum-{{ rum_release }} + target: install + params: + USE_PGXS: 1 + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/17-pg_hashids.yml b/ansible/tasks/postgres-extensions/17-pg_hashids.yml new file mode 100644 index 000000000..5d1a6e4cd --- /dev/null +++ b/ansible/tasks/postgres-extensions/17-pg_hashids.yml @@ -0,0 +1,17 @@ +# pg_hashids +- name: pg_hashids - download from master branch + git: + repo: https://github.com/iCyberon/pg_hashids.git + dest: /tmp/pg_hashids + version: master + +- name: pg_hashids - build + make: + chdir: /tmp/pg_hashids + become: yes + +- name: pg_hashids - install + make: + chdir: /tmp/pg_hashids + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 18dde1e42..f9bad4c57 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -41,4 +41,10 @@ import_tasks: tasks/postgres-extensions/14-pg_plan_filter.yml - name: Install pg_net - import_tasks: tasks/postgres-extensions/15-pg_net.yml \ No newline at end of file + import_tasks: tasks/postgres-extensions/15-pg_net.yml + +- name: Install rum + import_tasks: tasks/postgres-extensions/16-rum.yml + +- name: Install pg_hashids + import_tasks: tasks/postgres-extensions/17-pg_hashids.yml \ No newline at end of file From 25c93dedeacd4c5748611cb2df4c904ea4ea8a9d Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:11:40 +0800 Subject: [PATCH 08/29] Cleanup build dependencies immediately after installation --- ansible/tasks/postgres-extensions/01-postgis.yml | 16 +++++++++++++++- .../tasks/postgres-extensions/02-pgrouting.yml | 16 +++++++++++++++- ansible/tasks/postgres-extensions/05-pgaudit.yml | 9 ++++++++- .../tasks/postgres-extensions/07-pgsql-http.yml | 8 +++++++- .../postgres-extensions/08-plpgsql_check.yml | 8 +++++++- ansible/tasks/postgres-extensions/12-pljava.yml | 4 +++- ansible/tasks/postgres-extensions/13-plv8.yml | 10 +++++++++- ansible/tasks/postgres-extensions/15-pg_net.yml | 15 ++++++++++++++- ansible/tasks/setup-pgbouncer.yml | 11 ++++++++++- ansible/tasks/setup-postgres.yml | 14 ++++++++++++++ 10 files changed, 102 insertions(+), 9 deletions(-) diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml index 9aaad6d5e..4fb4df155 100644 --- a/ansible/tasks/postgres-extensions/01-postgis.yml +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -74,4 +74,18 @@ make: chdir: /tmp/postgis-{{ postgis_release }} target: install - become: yes \ No newline at end of file + become: yes + +- name: postgis - remove build dependencies + apt: + pkg: + - libgeos-dev + - libproj-dev + - libgdal-dev + - libjson-c-dev + - libxml2-dev + - libboost-all-dev + - libcgal-dev + - libmpfr-dev + - libgmp-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml index 9020d0024..6fe5c0c47 100644 --- a/ansible/tasks/postgres-extensions/02-pgrouting.yml +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -1,4 +1,12 @@ # pgRouting +- name: pgRouting - download & install dependencies + apt: + pkg: + - libboost-all-dev + update_cache: yes + cache_valid_time: 3600 + install_recommends: no + - name: pgRouting - download latest release get_url: url: "https://github.com/pgRouting/pgrouting/releases/download/v%7B%7B%20pgrouting_release%20%7D%7D/pgrouting-%7B%7B%20pgrouting_release%20%7D%7D.tar.gz" @@ -33,4 +41,10 @@ make: chdir: /tmp/pgrouting-{{ pgrouting_release }}/build target: install - become: yes \ No newline at end of file + become: yes + +- name: pgRouting - remove build dependencies + apt: + pkg: + - libboost-all-dev + state: absent diff --git a/ansible/tasks/postgres-extensions/05-pgaudit.yml b/ansible/tasks/postgres-extensions/05-pgaudit.yml index 6d3b2bca7..17937a35e 100644 --- a/ansible/tasks/postgres-extensions/05-pgaudit.yml +++ b/ansible/tasks/postgres-extensions/05-pgaudit.yml @@ -34,4 +34,11 @@ target: install params: USE_PGXS: 1 - become: yes \ No newline at end of file + become: yes + +- name: pgAudit - remove build dependencies + apt: + pkg: + - libssl-dev + - libkrb5-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml index 6fd5cf9aa..96e93fe53 100644 --- a/ansible/tasks/postgres-extensions/07-pgsql-http.yml +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -34,4 +34,10 @@ make: chdir: /tmp/pgsql-http-{{ pgsql_http_release }} target: install - become: yes \ No newline at end of file + become: yes + +- name: pgsql-http - remove build dependencies + apt: + pkg: + - libcurl4-gnutls-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml index 16fb5aa7e..2961c921c 100644 --- a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml +++ b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml @@ -29,4 +29,10 @@ make: chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} target: install - become: yes \ No newline at end of file + become: yes + +- name: plpgsql_check - remove build dependencies + apt: + pkg: + - libicu-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index 3bea59cb9..4f4c6a545 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -5,6 +5,7 @@ - maven - default-jre - default-jdk + - libssl-dev update_cache: yes install_recommends: no @@ -30,7 +31,7 @@ - name: pljava - install become: yes shell: - cmd: java -jar pljava-packaging/target/pljava-pg13.jar + cmd: java -jar pljava-packaging/target/pljava-pg{{ postgresql_major }}.jar chdir: /tmp/pljava-{{ pljava_release }} - name: pljava - remove build dependencies @@ -39,6 +40,7 @@ - maven - default-jre - default-jdk + - libssl-dev state: absent - name: pljava - install headless jdk diff --git a/ansible/tasks/postgres-extensions/13-plv8.yml b/ansible/tasks/postgres-extensions/13-plv8.yml index 36c11a449..481d3b24a 100644 --- a/ansible/tasks/postgres-extensions/13-plv8.yml +++ b/ansible/tasks/postgres-extensions/13-plv8.yml @@ -45,4 +45,12 @@ make: chdir: /tmp/plv8 target: install - become: yes \ No newline at end of file + become: yes + +- name: plv8 - remove build dependencies + apt: + pkg: + - libc++-dev + - libc++abi-dev + - libglib2.0-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/15-pg_net.yml b/ansible/tasks/postgres-extensions/15-pg_net.yml index 1bf7ae9bc..1e24062a3 100644 --- a/ansible/tasks/postgres-extensions/15-pg_net.yml +++ b/ansible/tasks/postgres-extensions/15-pg_net.yml @@ -1,4 +1,11 @@ # pg_net +- name: pg_net - download & install dependencies + apt: + pkg: + - libcurl4-gnutls-dev + update_cache: yes + install_recommends: no + - name: pg_net - download latest release get_url: url: "https://github.com/supabase/pg_net/archive/refs/tags/v%7B%7Bpg_net_release%7D%7D.tar.gz" @@ -21,4 +28,10 @@ make: chdir: /tmp/pg_net-{{ pg_net_release }} target: install - become: yes \ No newline at end of file + become: yes + +- name: pg_net - remove build dependencies + apt: + pkg: + - libcurl4-gnutls-dev + state: absent \ No newline at end of file diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 7400b19d9..5bd02b70e 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -5,6 +5,7 @@ - libssl-dev - pkg-config - libevent-dev + - libsystemd-dev update_cache: yes cache_valid_time: 3600 @@ -108,4 +109,12 @@ - name: PgBouncer - reload systemd systemd: - daemon_reload: yes \ No newline at end of file + daemon_reload: yes + +- name: PgBouncer - remove build dependencies + apt: + pkg: + - libssl-dev + - pkg-config + - libevent-dev + - libsystemd-dev \ No newline at end of file diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index bc94b1ace..44e15e82e 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -79,6 +79,20 @@ chdir: /tmp/postgresql-{{ postgresql_release }} become: yes +# Cleanup build dependencies +- name: Initialize the database + apt: + pkg: + - libreadline-dev + - zlib1g-dev + - libxml2-dev + - libxslt-dev + - libssl-dev + - libsystemd-dev + - libpq-dev + - uuid-dev + state: absent + # Create postgres user - name: Create postgres user user: From bd8258bbc1d3d1e090405fd6c4abfec93eb54e7e Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:12:09 +0800 Subject: [PATCH 09/29] Remove hardcoded values for Postgres major --- ansible/files/supabase_facts.ini | 2 +- ansible/tasks/docker/cleanup.yml | 2 +- ansible/tasks/docker/setup.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/files/supabase_facts.ini b/ansible/files/supabase_facts.ini index abafc8fb0..28598cfcf 100644 --- a/ansible/files/supabase_facts.ini +++ b/ansible/files/supabase_facts.ini @@ -1,2 +1,2 @@ [general] -postgres_version=13 +postgres_version=14 diff --git a/ansible/tasks/docker/cleanup.yml b/ansible/tasks/docker/cleanup.yml index 2ccc2af85..4552d8030 100644 --- a/ansible/tasks/docker/cleanup.yml +++ b/ansible/tasks/docker/cleanup.yml @@ -7,7 +7,7 @@ - rsync - ca-certificates - build-essential - - postgresql-server-dev-13 + - postgresql-server-dev-{{ postgresql_major }} - curl - git-core - gpp diff --git a/ansible/tasks/docker/setup.yml b/ansible/tasks/docker/setup.yml index 70a54d243..d669f8870 100644 --- a/ansible/tasks/docker/setup.yml +++ b/ansible/tasks/docker/setup.yml @@ -7,7 +7,7 @@ - rsync - ca-certificates - build-essential - - postgresql-server-dev-13 + - postgresql-server-dev-{{ postgresql_major }} - curl - git-core - gpp From 8794e9fa57bf9d3d366fef336ea4935a636673fe Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:13:35 +0800 Subject: [PATCH 10/29] Update variables --- ansible/vars.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d76034f0d..f70cec6ad 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -16,9 +16,12 @@ postgrest_x86_release_checksum: sha1:4b4adde15f0d41d65a9136d1f8c0d9cd6fe79326 aws_cli_release: "2.2.7" -golang_version: "1.15.4" -wal_g_release: "0.2.15" -wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 +golang_version: "1.17.3" +golang_version_checksum: + arm64: sha256:06f505c8d27203f78706ad04e47050b49092f1b06dc9ac4fbee4f0e4d015c8d4 + amd64: sha256:550f9845451c0c94be679faf116291e7807a8d78b43149f9506c1b15eb89008c + +wal_g_release: "v1.1.1" sfcgal_release: "1.3.10" sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b @@ -60,16 +63,16 @@ pgsql_http_release_checksum: sha1:816a3fff53e05301b176cf0696799fc5a00f54e8 plpgsql_check_release: "2.0.5" plpgsql_check_release_checksum: sha1:7fc9181f291bb0b24a7886681ab8bb837041ab62 -pg_safeupdate_release: "1.3" -pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c +pg_safeupdate_release: "1.4" +pg_safeupdate_release_checksum: sha1:942dacd0ebce6123944212ffb3d6b5a0c09174f9 timescaledb_release: "2.3.0" wal2json_release: "2_4" wal2json_release_checksum: sha1:9f5c8dec3c0a5c19b1b77273f67fd9b0b0ab6664 -supautils_release: "1.1.1" -supautils_release_checksum: sha1:431e1be36011026c7765dc365fa36cd5d6162f11 +supautils_release: "1.1.2" +supautils_release_checksum: sha1:2f224207b08bf629b21a0d7f4c236360821ee5c6 pljava_release: "1_6_3" pljava_release_checksum: sha1:550bea791c404c9d62050fd9c330e162bab20763 @@ -81,5 +84,8 @@ pg_plan_filter_commit_version: 5081a7b5cb890876e67d8e7486b6a64c38c9a492 pg_net_release: "0.3" pg_net_release_checksum: sha1:0695ad2e4e9b2a35a77dfa89bd4c5a90c622eb24 +rum_release: "1.3.9" +rum_release_checksum: sha1:71901640ccf9e2e1886aad37703a9fd07ced9e53 + vector_x86_deb: 'https://packages.timber.io/vector/0.17.0/vector-0.17.0-amd64.deb' vector_arm_deb: 'https://packages.timber.io/vector/0.17.0/vector-0.17.0-arm64.deb' From a21dd05ce042b1f61fd5534c5defd87e42fc45a6 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 16:13:51 +0800 Subject: [PATCH 11/29] Update wal-g installation process --- ansible/tasks/setup-wal-g.yml | 52 +++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 3f2bc5adc..4e1c71242 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,9 +1,10 @@ # Downloading dependencies -- name: Postgres dependencies +- name: wal-g dependencies become: yes apt: pkg: - liblzo2-dev + - libsodium-dev - cmake # install go dependency for WAL-G @@ -11,6 +12,7 @@ get_url: url: "https://golang.org/dl/go%7B%7B%20golang_version%20%7D%7D.linux-%7B%7B%20platform%20%7D%7D.tar.gz" dest: /tmp + checksum: "{{ golang_version_checksum[platform] }}" - name: unpack go archive unarchive: @@ -19,23 +21,43 @@ dest: /usr/local # Download WAL-G -- name: download wal-g - shell: - cmd: go get github.com/wal-g/wal-g; - environment: - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" +- name: wal-g - download latest version + git: + repo: https://github.com/wal-g/wal-g.git + dest: /tmp/wal-g + version: "{{ wal_g_release }}" + become: yes + +- name: wal-g - additional go dependencies + make: + chdir: /tmp/wal-g + target: deps + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true + become: yes ignore_errors: yes - # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 -# Install WAL-G -- name: install wal-g +- name: wal-g - build + make: + chdir: /tmp/wal-g + target: pg_build + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true + become: yes + +- name: wal-g - install + make: + chdir: /tmp/wal-g + target: pg_install + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true become: yes - shell: - cmd: make install && make deps && make pg_install - chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" - environment: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" # Clean up Go - name: Uninstall Go From ec84a58017d0b90edadf853d593e134f5576e024 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 20:02:33 +0800 Subject: [PATCH 12/29] add build dependencies for supautils --- ansible/tasks/internal/supautils.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml index 04b47bb3a..022382bfb 100644 --- a/ansible/tasks/internal/supautils.yml +++ b/ansible/tasks/internal/supautils.yml @@ -1,4 +1,12 @@ # supautils +- name: supautils - download & install dependencies + apt: + pkg: + - build-essential + - clang-11 + update_cache: yes + cache_valid_time: 3600 + - name: supautils - download latest release get_url: url: "https://github.com/supabase/supautils/archive/refs/tags/v%7B%7B%20supautils_release%20%7D%7D.tar.gz" @@ -43,3 +51,10 @@ path: /etc/postgresql/postgresql.conf regexp: session_preload_libraries = '' replace: session_preload_libraries = 'supautils' + +- name: supautils - remove build dependencies + apt: + pkg: + - build-essential + - clang-11 + state: absent \ No newline at end of file From 43a1a1183498b8bf202729bbb867c5447382af05 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 18 Nov 2021 20:03:24 +0800 Subject: [PATCH 13/29] changes to pgbouncer config - use scram-sha-256 encryption - list out additional build dependencies - assign postgres as stats user --- ansible/files/pgbouncer_config/pgbouncer.ini.j2 | 2 +- ansible/tasks/setup-pgbouncer.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 index 6ebce5ca6..21d90e236 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -131,7 +131,7 @@ auth_query = SELECT * FROM pgbouncer.get_auth($1) admin_users = pgbouncer ;; comma-separated list of users who are just allowed to use SHOW command -stats_users = pgbouncer +stats_users = pgbouncer,postgres ;;; ;;; Pooler personality questions diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 5bd02b70e..942710515 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -2,6 +2,7 @@ - name: PgBouncer - download & install dependencies apt: pkg: + - build-essential - libssl-dev - pkg-config - libevent-dev @@ -72,7 +73,7 @@ insertafter: '# Default:' line: "{{ item }}" with_items: - - "host all pgbouncer 127.0.0.1/32 md5" + - "host all pgbouncer 127.0.0.1/32 scram-sha-256" - "# Allow connection by pgbouncer user" # Run PgBouncer SQL script @@ -114,7 +115,9 @@ - name: PgBouncer - remove build dependencies apt: pkg: + - build-essential - libssl-dev - pkg-config - libevent-dev - - libsystemd-dev \ No newline at end of file + - libsystemd-dev + state: absent \ No newline at end of file From adcbfb64556731dc1fcd5798ca9964dfad73f119 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Fri, 19 Nov 2021 09:53:06 +0800 Subject: [PATCH 14/29] Revert "Cleanup build dependencies immediately after installation" This reverts commit 25c93dedeacd4c5748611cb2df4c904ea4ea8a9d. --- ansible/tasks/postgres-extensions/01-postgis.yml | 16 +--------------- .../tasks/postgres-extensions/02-pgrouting.yml | 16 +--------------- ansible/tasks/postgres-extensions/05-pgaudit.yml | 9 +-------- .../tasks/postgres-extensions/07-pgsql-http.yml | 8 +------- .../postgres-extensions/08-plpgsql_check.yml | 8 +------- ansible/tasks/postgres-extensions/12-pljava.yml | 4 +--- ansible/tasks/postgres-extensions/13-plv8.yml | 10 +--------- ansible/tasks/postgres-extensions/15-pg_net.yml | 15 +-------------- ansible/tasks/setup-pgbouncer.yml | 11 ----------- ansible/tasks/setup-postgres.yml | 14 -------------- 10 files changed, 8 insertions(+), 103 deletions(-) diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml index 4fb4df155..9aaad6d5e 100644 --- a/ansible/tasks/postgres-extensions/01-postgis.yml +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -74,18 +74,4 @@ make: chdir: /tmp/postgis-{{ postgis_release }} target: install - become: yes - -- name: postgis - remove build dependencies - apt: - pkg: - - libgeos-dev - - libproj-dev - - libgdal-dev - - libjson-c-dev - - libxml2-dev - - libboost-all-dev - - libcgal-dev - - libmpfr-dev - - libgmp-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml index 6fe5c0c47..9020d0024 100644 --- a/ansible/tasks/postgres-extensions/02-pgrouting.yml +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -1,12 +1,4 @@ # pgRouting -- name: pgRouting - download & install dependencies - apt: - pkg: - - libboost-all-dev - update_cache: yes - cache_valid_time: 3600 - install_recommends: no - - name: pgRouting - download latest release get_url: url: "https://github.com/pgRouting/pgrouting/releases/download/v%7B%7B%20pgrouting_release%20%7D%7D/pgrouting-%7B%7B%20pgrouting_release%20%7D%7D.tar.gz" @@ -41,10 +33,4 @@ make: chdir: /tmp/pgrouting-{{ pgrouting_release }}/build target: install - become: yes - -- name: pgRouting - remove build dependencies - apt: - pkg: - - libboost-all-dev - state: absent + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/05-pgaudit.yml b/ansible/tasks/postgres-extensions/05-pgaudit.yml index 17937a35e..6d3b2bca7 100644 --- a/ansible/tasks/postgres-extensions/05-pgaudit.yml +++ b/ansible/tasks/postgres-extensions/05-pgaudit.yml @@ -34,11 +34,4 @@ target: install params: USE_PGXS: 1 - become: yes - -- name: pgAudit - remove build dependencies - apt: - pkg: - - libssl-dev - - libkrb5-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml index 96e93fe53..6fd5cf9aa 100644 --- a/ansible/tasks/postgres-extensions/07-pgsql-http.yml +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -34,10 +34,4 @@ make: chdir: /tmp/pgsql-http-{{ pgsql_http_release }} target: install - become: yes - -- name: pgsql-http - remove build dependencies - apt: - pkg: - - libcurl4-gnutls-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml index 2961c921c..16fb5aa7e 100644 --- a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml +++ b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml @@ -29,10 +29,4 @@ make: chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} target: install - become: yes - -- name: plpgsql_check - remove build dependencies - apt: - pkg: - - libicu-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index 4f4c6a545..3bea59cb9 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -5,7 +5,6 @@ - maven - default-jre - default-jdk - - libssl-dev update_cache: yes install_recommends: no @@ -31,7 +30,7 @@ - name: pljava - install become: yes shell: - cmd: java -jar pljava-packaging/target/pljava-pg{{ postgresql_major }}.jar + cmd: java -jar pljava-packaging/target/pljava-pg13.jar chdir: /tmp/pljava-{{ pljava_release }} - name: pljava - remove build dependencies @@ -40,7 +39,6 @@ - maven - default-jre - default-jdk - - libssl-dev state: absent - name: pljava - install headless jdk diff --git a/ansible/tasks/postgres-extensions/13-plv8.yml b/ansible/tasks/postgres-extensions/13-plv8.yml index 481d3b24a..36c11a449 100644 --- a/ansible/tasks/postgres-extensions/13-plv8.yml +++ b/ansible/tasks/postgres-extensions/13-plv8.yml @@ -45,12 +45,4 @@ make: chdir: /tmp/plv8 target: install - become: yes - -- name: plv8 - remove build dependencies - apt: - pkg: - - libc++-dev - - libc++abi-dev - - libglib2.0-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/15-pg_net.yml b/ansible/tasks/postgres-extensions/15-pg_net.yml index 1e24062a3..1bf7ae9bc 100644 --- a/ansible/tasks/postgres-extensions/15-pg_net.yml +++ b/ansible/tasks/postgres-extensions/15-pg_net.yml @@ -1,11 +1,4 @@ # pg_net -- name: pg_net - download & install dependencies - apt: - pkg: - - libcurl4-gnutls-dev - update_cache: yes - install_recommends: no - - name: pg_net - download latest release get_url: url: "https://github.com/supabase/pg_net/archive/refs/tags/v%7B%7Bpg_net_release%7D%7D.tar.gz" @@ -28,10 +21,4 @@ make: chdir: /tmp/pg_net-{{ pg_net_release }} target: install - become: yes - -- name: pg_net - remove build dependencies - apt: - pkg: - - libcurl4-gnutls-dev - state: absent \ No newline at end of file + become: yes \ No newline at end of file diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 942710515..610fb6f73 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -6,7 +6,6 @@ - libssl-dev - pkg-config - libevent-dev - - libsystemd-dev update_cache: yes cache_valid_time: 3600 @@ -111,13 +110,3 @@ - name: PgBouncer - reload systemd systemd: daemon_reload: yes - -- name: PgBouncer - remove build dependencies - apt: - pkg: - - build-essential - - libssl-dev - - pkg-config - - libevent-dev - - libsystemd-dev - state: absent \ No newline at end of file diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 44e15e82e..bc94b1ace 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -79,20 +79,6 @@ chdir: /tmp/postgresql-{{ postgresql_release }} become: yes -# Cleanup build dependencies -- name: Initialize the database - apt: - pkg: - - libreadline-dev - - zlib1g-dev - - libxml2-dev - - libxslt-dev - - libssl-dev - - libsystemd-dev - - libpq-dev - - uuid-dev - state: absent - # Create postgres user - name: Create postgres user user: From 9b82e064bbf9f497914d7bf2120a6feba30dc31a Mon Sep 17 00:00:00 2001 From: dragarcia Date: Fri, 19 Nov 2021 10:06:30 +0800 Subject: [PATCH 15/29] Minimise packages cleaned up - some extensions were dependent to them --- ansible/tasks/postgres-extensions/02-pgrouting.yml | 8 ++++++++ ansible/tasks/postgres-extensions/12-pljava.yml | 3 ++- ansible/tasks/postgres-extensions/15-pg_net.yml | 7 +++++++ ansible/tasks/setup-pgbouncer.yml | 1 + ansible/tasks/setup-postgres.yml | 1 - 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml index 9020d0024..ea995fc3e 100644 --- a/ansible/tasks/postgres-extensions/02-pgrouting.yml +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -1,4 +1,12 @@ # pgRouting +- name: pgRouting - download & install dependencies + apt: + pkg: + - libboost-all-dev + update_cache: yes + cache_valid_time: 3600 + install_recommends: no + - name: pgRouting - download latest release get_url: url: "https://github.com/pgRouting/pgrouting/releases/download/v%7B%7B%20pgrouting_release%20%7D%7D/pgrouting-%7B%7B%20pgrouting_release%20%7D%7D.tar.gz" diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index 3bea59cb9..8330728cf 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -5,6 +5,7 @@ - maven - default-jre - default-jdk + - libssl-dev update_cache: yes install_recommends: no @@ -30,7 +31,7 @@ - name: pljava - install become: yes shell: - cmd: java -jar pljava-packaging/target/pljava-pg13.jar + cmd: java -jar pljava-packaging/target/pljava-pg{{ postgresql_major }}.jar chdir: /tmp/pljava-{{ pljava_release }} - name: pljava - remove build dependencies diff --git a/ansible/tasks/postgres-extensions/15-pg_net.yml b/ansible/tasks/postgres-extensions/15-pg_net.yml index 1bf7ae9bc..8f00f8508 100644 --- a/ansible/tasks/postgres-extensions/15-pg_net.yml +++ b/ansible/tasks/postgres-extensions/15-pg_net.yml @@ -1,4 +1,11 @@ # pg_net +- name: pg_net - download & install dependencies + apt: + pkg: + - libcurl4-gnutls-dev + update_cache: yes + install_recommends: no + - name: pg_net - download latest release get_url: url: "https://github.com/supabase/pg_net/archive/refs/tags/v%7B%7Bpg_net_release%7D%7D.tar.gz" diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 610fb6f73..ae429e188 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -6,6 +6,7 @@ - libssl-dev - pkg-config - libevent-dev + - libsystemd-dev update_cache: yes cache_valid_time: 3600 diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index bc94b1ace..8f76a3f4f 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -12,7 +12,6 @@ - libxslt-dev - libssl-dev - libsystemd-dev - - libpq-dev - libxml2-utils - uuid-dev - xsltproc From e2aab01c3b7c39af645c5c325fee9a3bd2920c35 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Fri, 19 Nov 2021 16:35:25 +0800 Subject: [PATCH 16/29] link pg binaries only at the end --- ansible/playbook.yml | 15 +++++++++++++++ ansible/tasks/setup-postgres.yml | 14 -------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 319cd8e2c..0e20df6c7 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -139,3 +139,18 @@ - unattended-upgrades update_cache: yes cache_valid_time: 3600 + + # Put PG binaries in a directory under $PATH + - name: Find all files in /usr/lib/postgresql/bin + find: + paths: /usr/lib/postgresql/bin + register: postgresql_bin + + - name: Create symbolic links for Postgres binaries to /usr/bin/ + become: yes + file: + src: "{{ item.path }}" + path: "/usr/bin/{{ item.path | basename }}" + state: link + force: yes + with_items: "{{ postgresql_bin.files }}" \ No newline at end of file diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 8f76a3f4f..3895a3823 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -132,20 +132,6 @@ dest: /etc/postgresql/pg_ident.conf group: postgres -- name: Find all files in /usr/lib/postgresql/bin - find: - paths: /usr/lib/postgresql/bin - register: postgresql_bin - -- name: Create symbolic links for Postgres binaries to /usr/bin/ - become: yes - file: - src: "{{ item.path }}" - path: "/usr/bin/{{ item.path | basename }}" - state: link - force: yes - with_items: "{{ postgresql_bin.files }}" - # init DB - name: Initialize the database become: yes From cf0f434791e35d9c45000d8dd6be743a7f560690 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Fri, 19 Nov 2021 16:36:46 +0800 Subject: [PATCH 17/29] remove postgres as pgbouncer stats_users --- ansible/files/pgbouncer_config/pgbouncer.ini.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 index 21d90e236..6ebce5ca6 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -131,7 +131,7 @@ auth_query = SELECT * FROM pgbouncer.get_auth($1) admin_users = pgbouncer ;; comma-separated list of users who are just allowed to use SHOW command -stats_users = pgbouncer,postgres +stats_users = pgbouncer ;;; ;;; Pooler personality questions From 60a6fdd87b706c7bd0e650def6315e86a131674d Mon Sep 17 00:00:00 2001 From: dragarcia Date: Fri, 19 Nov 2021 16:38:45 +0800 Subject: [PATCH 18/29] bump to Postgres 14.1 --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index f70cec6ad..f1dcafdef 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,8 +1,8 @@ supabase_internal: true postgresql_major: "14" -postgresql_release: "14.0" -postgresql_release_checksum: sha1:a58a5492b25d89c79e7a3727e09dd2847b2b3f33 +postgresql_release: "14.1" +postgresql_release_checksum: sha1:f6c114a4be41ca876deeb462e9d02237b186743c # Non Postgres Extensions pgbouncer_release: "1.16.0" From 1cf93514da7457d2275a0c240e2aa5bc8331d03b Mon Sep 17 00:00:00 2001 From: dragarcia Date: Sun, 21 Nov 2021 23:24:19 +0800 Subject: [PATCH 19/29] rename filename for pgsodium --- .../postgres-extensions/{16-pgsodium.yml => 18-pgsodium.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ansible/tasks/postgres-extensions/{16-pgsodium.yml => 18-pgsodium.yml} (100%) diff --git a/ansible/tasks/postgres-extensions/16-pgsodium.yml b/ansible/tasks/postgres-extensions/18-pgsodium.yml similarity index 100% rename from ansible/tasks/postgres-extensions/16-pgsodium.yml rename to ansible/tasks/postgres-extensions/18-pgsodium.yml From c98446cc457e867acbedbaa015038557fca5d8ae Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 22 Nov 2021 11:17:21 +0800 Subject: [PATCH 20/29] update regex for session_preload_libraries --- ansible/tasks/internal/supautils.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml index 022382bfb..d1ce96572 100644 --- a/ansible/tasks/internal/supautils.yml +++ b/ansible/tasks/internal/supautils.yml @@ -49,7 +49,7 @@ become: yes replace: path: /etc/postgresql/postgresql.conf - regexp: session_preload_libraries = '' + regexp: "#session_preload_libraries = ''" replace: session_preload_libraries = 'supautils' - name: supautils - remove build dependencies From 9feea2173534ca5ee521645c43d6d1643bbff822 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 22 Nov 2021 11:57:18 +0800 Subject: [PATCH 21/29] Update README --- README.md | 55 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 528b7385a..7be145c8a 100644 --- a/README.md +++ b/README.md @@ -3,32 +3,35 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to modify Postgres, but to provide some of the most common extensions with a one-click install. ## Primary Features -- ✅ Postgres [13](https://www.postgresql.org/about/news/postgresql-13-released-2077/). +- ✅ Postgres [14](https://www.postgresql.org/about/news/postgresql-14-released-2318/). - ✅ Ubuntu 20.04 (Focal Fossa). - ✅ [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html) = logical and [max_replication_slots](https://www.postgresql.org/docs/current/runtime-config-replication.html) = 5. Ready for replication. - ✅ [Large Systems Extensions](https://github.com/aws/aws-graviton-getting-started#building-for-graviton-and-graviton2). Enabled for ARM images. ## Extensions -| Extension | Description | -| ------------- | ------------- | -| [Postgres contrib modules](https://www.postgresql.org/docs/current/contrib.html) | Because everyone should enable `pg_stat_statements`. | -| [PostGIS](https://postgis.net/) | Postgres' most popular extension - support for geographic objects. | -| [pgRouting](https://pgrouting.org/) | Extension of PostGIS - provides geospatial routing functionalities. | -| [pgTAP](https://pgtap.org/) | Unit Testing for Postgres. | -| [pg_cron](https://github.com/citusdata/pg_cron) | Run CRON jobs inside Postgres. | -| [pgAudit](https://www.pgaudit.org/) | Generate highly compliant audit logs. | -| [pgjwt](https://github.com/michelp/pgjwt) | Generate JSON Web Tokens (JWT) in Postgres. | -| [pgsql-http](https://github.com/pramsey/pgsql-http) | HTTP client for Postgres. | -| [plpgsql_check](https://github.com/okbob/plpgsql_check) | Linter tool for PL/pgSQL. | -| [pg-safeupdate](https://github.com/eradman/pg-safeupdate) | Protect your data from accidental updates or deletes. | -| [wal2json](https://github.com/eulerto/wal2json) | JSON output plugin for logical replication decoding. | -| [PL/Java](https://github.com/tada/pljava) | Write in Java functions in Postgres. | -| [plv8](https://github.com/plv8/plv8) | Write in Javascript functions in Postgres. | -| [pg_plan_filter](https://github.com/pgexperts/pg_plan_filter) | Only allow statements that fulfill set criteria to be executed. | -| [pg_net](https://github.com/supabase/pg_net) | Expose the SQL interface for async networking. | -| [pg_sodium](https://github.com/michelp/pgsodium) | Modern encryption API using libsodium. | - -Can't find your favorite extension? Suggest for it to be added into future versions [here](https://github.com/supabase/supabase/discussions/679)! +| Extension | Version | Description | +| ------------- | :-------------: | ------------- | +| [Postgres contrib modules](https://www.postgresql.org/docs/current/contrib.html) | - | Because everyone should enable `pg_stat_statements`. | +| [PostGIS](https://postgis.net/) | [3.1.4](https://git.osgeo.org/gitea/postgis/postgis/raw/tag/3.1.4/NEWS) | Postgres' most popular extension - support for geographic objects. | +| [pgRouting](https://pgrouting.org/) | [v3.3.0](https://github.com/pgRouting/pgrouting/releases/tag/v3.3.0) | Extension of PostGIS - provides geospatial routing functionalities. | +| [pgTAP](https://pgtap.org/) | [v1.1.0](https://github.com/theory/pgtap/releases/tag/v1.1.0) | Unit Testing for Postgres. | +| [pg_cron](https://github.com/citusdata/pg_cron) | [v1.4.1](https://github.com/citusdata/pg_cron/releases/tag/v1.4.1) | Run CRON jobs inside Postgres. | +| [pgAudit](https://www.pgaudit.org/) | [1.6.1](https://github.com/pgaudit/pgaudit/releases/tag/1.6.1) | Generate highly compliant audit logs. | +| [pgjwt](https://github.com/michelp/pgjwt) | [commit](https://github.com/michelp/pgjwt/commit/9742dab1b2f297ad3811120db7b21451bca2d3c9) | Generate JSON Web Tokens (JWT) in Postgres. | +| [pgsql-http](https://github.com/pramsey/pgsql-http) | [1.3.1](https://github.com/pramsey/pgsql-http/releases/tag/v1.3.1) | HTTP client for Postgres. | +| [plpgsql_check](https://github.com/okbob/plpgsql_check) | [2.0.6](https://github.com/okbob/plpgsql_check/releases/tag/v2.0.6) | Linter tool for PL/pgSQL. | +| [pg-safeupdate](https://github.com/eradman/pg-safeupdate) | [1.4](https://github.com/eradman/pg-safeupdate/releases/tag/1.4) | Protect your data from accidental updates or deletes. | +| [wal2json](https://github.com/eulerto/wal2json) | [2.4](https://github.com/eulerto/wal2json/releases/tag/wal2json_2_4) | JSON output plugin for logical replication decoding. | +| [PL/Java](https://github.com/tada/pljava) | [1.6.3](https://github.com/tada/pljava/releases/tag/V1_6_3) | Write in Java functions in Postgres. | +| [plv8](https://github.com/plv8/plv8) | [commit](https://github.com/plv8/plv8/commit/3656177d384e3e02b74faa8e2931600f3690ab59) | Write in Javascript functions in Postgres. | +| [pg_plan_filter](https://github.com/pgexperts/pg_plan_filter) | [commit](https://github.com/pgexperts/pg_plan_filter/commit/5081a7b5cb890876e67d8e7486b6a64c38c9a492) | Only allow statements that fulfill set criteria to be executed. | +| [pg_net](https://github.com/supabase/pg_net) | [v0.3](https://github.com/supabase/pg_net/releases/tag/v0.3) | Expose the SQL interface for async networking. | +| [rum](https://github.com/postgrespro/rum) | [1.3.9](https://github.com/postgrespro/rum/releases/tag/1.3.9) | An alternative to the GIN index. | +| [pg_hashids](https://github.com/iCyberon/pg_hashids) | [commit](https://github.com/iCyberon/pg_hashids/commit/83398bcbb616aac2970f5e77d93a3200f0f28e74) | Generate unique identifiers from numbers. | +| [pg_sodium](https://github.com/michelp/pgsodium) | [v1.3.0](https://github.com/michelp/pgsodium/releases/tag/v1.3.0) | Modern encryption API using libsodium. | + + +Can't find your favorite extension? Suggest for it to be added into future releases [here](https://github.com/supabase/supabase/discussions/679)! ## Enhanced Security *This is only available for our AWS EC2/ DO Droplet images* @@ -42,11 +45,11 @@ Aside from having [ufw](https://help.ubuntu.com/community/UFW),[fail2ban](https: ## Additional Goodies *This is only available for our AWS EC2/ DO Droplet images* -| Goodie | Description | -| ------------- | ------------- | -| [PgBouncer](https://www.pgbouncer.org/) | Set up Connection Pooling. | -| [PostgREST](https://postgrest.org/en/stable/) | Instantly transform your database into an RESTful API. | -| [WAL-G](https://github.com/wal-g/wal-g#wal-g) | Tool for physical database backup and recovery. | +| Goodie | Version | Description | +| ------------- | :-------------: | ------------- | +| [PgBouncer](https://www.pgbouncer.org/) | [1.16.1](http://www.pgbouncer.org/changelog.html#pgbouncer-116x) | Set up Connection Pooling. | +| [PostgREST](https://postgrest.org/en/stable/) | [v8.0.0](https://github.com/PostgREST/postgrest/releases/tag/v8.0.0) | Instantly transform your database into an RESTful API. | +| [WAL-G](https://github.com/wal-g/wal-g#wal-g) | [v1.1](https://github.com/wal-g/wal-g/releases/tag/v1.1) | Tool for physical database backup and recovery. | ## Install From 33d7500168ce5f6e81f497ed56e5f2110a425fda Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 22 Nov 2021 12:05:35 +0800 Subject: [PATCH 22/29] update download URLs for postgrest --- ansible/tasks/setup-postgrest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 16a10e8c7..cfd0647a1 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -10,14 +10,14 @@ - name: PostgREST - download ubuntu binary archive (arm) get_url: - url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-%7B%7B%20postgrest_arm_release%20%7D%7D.tar.xz" + url: "https://github.com/PostgREST/postgrest/releases/download/v%7B%7B%20postgrest_release%20%7D%7D/postgrest-v%7B%7B%20postgrest_release%20%7D%7D-linux-x64-static.tar.xz" dest: /tmp/postgrest.tar.xz checksum: "{{ postgrest_arm_release_checksum }}" when: platform == "arm64" - name: PostgREST - download ubuntu binary archive (x86) get_url: - url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-%7B%7B%20postgrest_x86_release%20%7D%7D.tar.xz" + url: "https://github.com/PostgREST/postgrest/releases/download/v%7B%7B%20postgrest_release%20%7D%7D/postgrest-v%7B%7B%20postgrest_release%20%7D%7D-ubuntu-aarch64.tar.xz" dest: /tmp/postgrest.tar.xz checksum: "{{ postgrest_x86_release_checksum }}" when: platform == "amd64" From 5bf900bc43910d8d2bce9abd65268ba8a1529554 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 22 Nov 2021 12:05:46 +0800 Subject: [PATCH 23/29] Bump versions --- ansible/vars.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a51b706aa..04be3aa90 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -5,14 +5,12 @@ postgresql_release: "14.1" postgresql_release_checksum: sha1:f6c114a4be41ca876deeb462e9d02237b186743c # Non Postgres Extensions -pgbouncer_release: "1.16.0" -pgbouncer_release_checksum: sha1:7d4b7e1110387df2245b22de3168884fdc8092c4 +pgbouncer_release: "1.16.1" +pgbouncer_release_checksum: sha1:14c75af0b5a11b0363b6146170b516db498fc998 -postgrest_arm_release: 2021-03-05-19-03-d3a8b5f-ubuntu-aarch64 -postgrest_arm_release_checksum: sha1:b9e9b06ead7230b75033e8ae17912714bf463a33 - -postgrest_x86_release: 2021-03-05-19-03-d3a8b5f-linux-x64-static -postgrest_x86_release_checksum: sha1:4b4adde15f0d41d65a9136d1f8c0d9cd6fe79326 +postgrest_release: "8.0.0" +postgrest_arm_release_checksum: sha1:dfd3b88b3acddd0ede52c6bed2bb7c02cbcac5dd +postgrest_x86_release_checksum: sha1:29a49bb898d5008b746822acdbaa4fabb51b5c44 aws_cli_release: "2.2.7" @@ -21,7 +19,7 @@ golang_version_checksum: arm64: sha256:06f505c8d27203f78706ad04e47050b49092f1b06dc9ac4fbee4f0e4d015c8d4 amd64: sha256:550f9845451c0c94be679faf116291e7807a8d78b43149f9506c1b15eb89008c -wal_g_release: "v1.1.1" +wal_g_release: "v1.1" sfcgal_release: "1.3.10" sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b @@ -45,8 +43,8 @@ adminapi_release_checksum: postgis_release: "3.1.4" postgis_release_checksum: sha1:3077da5136841d9d51a4325a233be8eccf763c38 -pgrouting_release: "3.2.2" -pgrouting_release_checksum: sha1:56596fa6e22104572d61296e1fa25e7918c79671 +pgrouting_release: "3.3.0" +pgrouting_release_checksum: sha1:7475189f334bc97e1b3587b2f456a573c8aa811a pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e @@ -60,8 +58,8 @@ pgaudit_release_checksum: sha1:2fae2690d38a1822b6e8f40ad6b2bd8669498659 pgsql_http_release: "1.3.1" pgsql_http_release_checksum: sha1:816a3fff53e05301b176cf0696799fc5a00f54e8 -plpgsql_check_release: "2.0.5" -plpgsql_check_release_checksum: sha1:7fc9181f291bb0b24a7886681ab8bb837041ab62 +plpgsql_check_release: "2.0.6" +plpgsql_check_release_checksum: sha1:3f3ca49e38ee87392fd62f1a62a481c445e1c6c2 pg_safeupdate_release: "1.4" pg_safeupdate_release_checksum: sha1:942dacd0ebce6123944212ffb3d6b5a0c09174f9 From ceae00865b7a7c1742a3002d2ad413732b9ade89 Mon Sep 17 00:00:00 2001 From: Div Arora Date: Mon, 22 Nov 2021 20:44:22 +0530 Subject: [PATCH 24/29] feat: build AMIs using Github Actions (#101) * feat: build AMIs using Github Actions * eliminate two-step build Co-authored-by: dragarcia --- .github/workflows/ci.yml | 19 +++++++++++++++++++ amazon.json | 34 ++++++++++++++++++---------------- common.vars.json | 3 +++ development-arm.vars.json | 8 ++++++++ 4 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 common.vars.json create mode 100644 development-arm.vars.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..892e269ec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,19 @@ +name: Run CI checks + +on: + push: + branches: + - develop + +jobs: + build: + runs-on: [self-hosted, linux] + timeout-minutes: 150 + + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + + - name: Build AMI + run: | + packer build -timestamp-ui -color=false -on-error=abort -var-file common.vars.json -var-file development-arm.vars.json amazon.json diff --git a/amazon.json b/amazon.json index 5a57dc871..63fb266b3 100644 --- a/amazon.json +++ b/amazon.json @@ -1,34 +1,36 @@ { "variables": { - "aws_access_key": "", - "aws_secret_key": "", - "region": "af-south-1", - "ami_regions": "af-south-1", - "ami": "ami-08a4b40f2fe1e4b35", - "ami_name": "supabase-postgres-13.3.0.4", - "environment": "prod", - "ansible_arguments": "--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal" + "profile": "{{env `AWS_PROFILE`}}" }, "builders": [ { "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", + "profile": "{{user `profile`}}", "region": "{{user `region`}}", "ami_regions": "{{user `ami_regions`}}", - "source_ami": "{{user `ami`}}", - "instance_type": "m5.2xlarge", + "source_ami": "{{user `ubuntu-2004`}}", + "instance_type": "{{user `instance-type`}}", "ssh_username": "ubuntu", - "ami_name": "{{user `ami_name`}}", + "ami_name": "supabase-postgres-{{user `postgres-version`}}", "tags": { "environment": "{{user `environment`}}", - "appType": "postgres" + "appType": "postgres", + "creator": "packer" + }, + "run_tags": { + "creator": "packer" + }, + "snapshot_tags": { + "creator": "packer" + }, + "run_volume_tags": { + "creator": "packer" }, "launch_block_device_mappings": [ { "device_name": "/dev/sda1", "volume_size": 16, - "volume_type": "gp2", + "volume_type": "gp3", "delete_on_termination": true } ] @@ -45,7 +47,7 @@ "type": "ansible", "user": "ubuntu", "playbook_file": "ansible/playbook.yml", - "extra_arguments": "{{user `ansible_arguments`}}" + "extra_arguments": "--skip-tags,install-postgrest" }, { "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", diff --git a/common.vars.json b/common.vars.json new file mode 100644 index 000000000..83454511d --- /dev/null +++ b/common.vars.json @@ -0,0 +1,3 @@ +{ + "postgres-version": "13.3.0.5-rc0" +} diff --git a/development-arm.vars.json b/development-arm.vars.json new file mode 100644 index 000000000..d0c6f99f6 --- /dev/null +++ b/development-arm.vars.json @@ -0,0 +1,8 @@ +{ + "ami_regions": "ap-southeast-1", + "arch": "arm64", + "environment": "dev", + "instance-type": "t4g.2xlarge", + "region": "ap-southeast-1", + "ubuntu-2004": "ami-077adae4d983338da" +} From ccd7a8127cd427636ea0f2fb4c4107ca426aaa42 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 22 Nov 2021 23:42:59 +0800 Subject: [PATCH 25/29] Transfer ownership of pgbouncer process to pgbouncer user --- .../files/pgbouncer_config/pgbouncer.service.j2 | 2 +- .../tmpfiles.d-pgbouncer.conf.j2 | 2 +- ansible/tasks/setup-pgbouncer.yml | 17 ++++++++++++++--- scripts/91-log_cleanup.sh | 4 +++- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/ansible/files/pgbouncer_config/pgbouncer.service.j2 b/ansible/files/pgbouncer_config/pgbouncer.service.j2 index 96273cb69..4ada2c690 100644 --- a/ansible/files/pgbouncer_config/pgbouncer.service.j2 +++ b/ansible/files/pgbouncer_config/pgbouncer.service.j2 @@ -30,7 +30,7 @@ After=network.target [Service] Type=notify -User=postgres +User=pgbouncer ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini ExecReload=/bin/kill -HUP $MAINPID KillSignal=SIGINT diff --git a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 index 3889ed294..d5d2cd49d 100644 --- a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 +++ b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 @@ -1,2 +1,2 @@ # Directory for PostgreSQL sockets, lockfiles and stats tempfiles -d /run/pgbouncer 2775 postgres postgres - - \ No newline at end of file +d /run/pgbouncer 2775 pgbouncer postgres - - \ No newline at end of file diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index ae429e188..3e805cc8d 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -40,24 +40,35 @@ target: install become: yes +# Create pgbouncer user +- name: Create pgbouncer user + user: + name: pgbouncer + shell: /bin/false + comment: PgBouncer user + groups: postgres,ssl-cert + # Create /etc/postgresql directory and make sure postgres group owns it - name: PgBouncer - create a directory if it does not exist file: path: /etc/pgbouncer state: directory - group: postgres + owner: pgbouncer + mode: '0700' - name: PgBouncer - adjust pgbouncer.ini copy: src: files/pgbouncer_config/pgbouncer.ini.j2 dest: /etc/pgbouncer/pgbouncer.ini + owner: pgbouncer + mode: '0700' - name: PgBouncer - create a directory if it does not exist file: path: /etc/pgbouncer/userlist.txt state: touch - group: postgres - owner: postgres + owner: pgbouncer + mode: '0700' - name: import /etc/tmpfiles.d/pgbouncer.conf template: diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh index e00dbb4f4..4c8441adb 100644 --- a/scripts/91-log_cleanup.sh +++ b/scripts/91-log_cleanup.sh @@ -8,5 +8,7 @@ rm -rf /var/log/* touch /var/log/auth.log touch /var/log/pgbouncer.log +chown pgbouncer:postgres /var/log/pgbouncer.log + mkdir /var/log/postgresql -chown postgres:postgres /var/log/pgbouncer.log /var/log/postgresql \ No newline at end of file +chown postgres:postgres /var/log/postgresql \ No newline at end of file From 6f615a150d25c6b65dc5766150d8aeb7c83d32b5 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 25 Nov 2021 13:41:53 +0800 Subject: [PATCH 26/29] increased timeout for get_url --- ansible/tasks/internal/admin-api.yml | 1 + ansible/tasks/internal/node-exporter.yml | 1 + ansible/tasks/internal/postgres-exporter.yml | 1 + ansible/tasks/internal/supautils.yml | 1 + ansible/tasks/postgres-extensions/01-postgis.yml | 2 ++ ansible/tasks/postgres-extensions/02-pgrouting.yml | 1 + ansible/tasks/postgres-extensions/03-pgtap.yml | 1 + ansible/tasks/postgres-extensions/04-pg_cron.yml | 1 + ansible/tasks/postgres-extensions/05-pgaudit.yml | 1 + ansible/tasks/postgres-extensions/07-pgsql-http.yml | 1 + ansible/tasks/postgres-extensions/08-plpgsql_check.yml | 1 + ansible/tasks/postgres-extensions/09-pg-safeupdate.yml | 1 + ansible/tasks/postgres-extensions/11-wal2json.yml | 1 + ansible/tasks/postgres-extensions/12-pljava.yml | 1 + ansible/tasks/postgres-extensions/15-pg_net.yml | 1 + ansible/tasks/postgres-extensions/16-rum.yml | 1 + ansible/tasks/postgres-extensions/18-pgsodium.yml | 2 ++ ansible/tasks/setup-postgres.yml | 1 + ansible/tasks/setup-postgrest.yml | 2 ++ ansible/tasks/setup-supabase-internal.yml | 2 ++ ansible/tasks/setup-wal-g.yml | 1 + 21 files changed, 25 insertions(+) diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index 9b3cb0b6b..4a15c5aa7 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -32,6 +32,7 @@ url: "https://github.com/supabase/supabase-admin-api/releases/download/%7B%7B%20adminapi_release%20%7D%7D/supabase-admin-api-%7B%7B%20adminapi_release%20%7D%7D-%7B%7B%20arch%20%7D%7D.tar.gz" dest: /tmp/adminapi.tar.gz checksum: "{{ adminapi_release_checksum[platform] }}" + timeout: 60 - name: adminapi - unpack archive in /opt unarchive: diff --git a/ansible/tasks/internal/node-exporter.yml b/ansible/tasks/internal/node-exporter.yml index 355dcdb15..ef1eabcf9 100644 --- a/ansible/tasks/internal/node-exporter.yml +++ b/ansible/tasks/internal/node-exporter.yml @@ -8,6 +8,7 @@ url: "https://github.com/prometheus/node_exporter/releases/download/v%7B%7B%20node_exporter_release%20%7D%7D/node_exporter-%7B%7B%20node_exporter_release%20%7D%7D.linux-%7B%7B%20platform%20%7D%7D.tar.gz" dest: /tmp/node_exporter.tar.gz checksum: "{{ node_exporter_release_checksum[platform] }}" + timeout: 60 - name: create directories file: diff --git a/ansible/tasks/internal/postgres-exporter.yml b/ansible/tasks/internal/postgres-exporter.yml index 49d9acff6..b4c1aedc1 100644 --- a/ansible/tasks/internal/postgres-exporter.yml +++ b/ansible/tasks/internal/postgres-exporter.yml @@ -19,6 +19,7 @@ url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v%7B%7B%20postgres_exporter_release%20%7D%7D/postgres_exporter-%7B%7B%20postgres_exporter_release%20%7D%7D.linux-%7B%7B%20platform%20%7D%7D.tar.gz" dest: /tmp/postgres_exporter.tar.gz checksum: "{{ postgres_exporter_release_checksum[platform] }}" + timeout: 60 - name: expand postgres exporter unarchive: diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml index d1ce96572..7422dbb56 100644 --- a/ansible/tasks/internal/supautils.yml +++ b/ansible/tasks/internal/supautils.yml @@ -12,6 +12,7 @@ url: "https://github.com/supabase/supautils/archive/refs/tags/v%7B%7B%20supautils_release%20%7D%7D.tar.gz" dest: /tmp/supautils-{{ supautils_release }}.tar.gz checksum: "{{ supautils_release_checksum }}" + timeout: 60 - name: supautils - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml index 9aaad6d5e..089087aab 100644 --- a/ansible/tasks/postgres-extensions/01-postgis.yml +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -21,6 +21,7 @@ url: "https://gitlab.com/Oslandia/SFCGAL/-/archive/v%7B%7B%20sfcgal_release%20%7D%7D/SFCGAL-v%7B%7B%20sfcgal_release%20%7D%7D.tar.gz" dest: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz checksum: "{{ sfcgal_release_checksum }}" + timeout: 60 - name: postgis - unpack SFCGAL unarchive: @@ -51,6 +52,7 @@ url: "https://download.osgeo.org/postgis/source/postgis-%7B%7B%20postgis_release%20%7D%7D.tar.gz" dest: /tmp/postgis-{{ postgis_release }}.tar.gz checksum: "{{ postgis_release_checksum }}" + timeout: 60 - name: postgis - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml index ea995fc3e..5d9ca879c 100644 --- a/ansible/tasks/postgres-extensions/02-pgrouting.yml +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -12,6 +12,7 @@ url: "https://github.com/pgRouting/pgrouting/releases/download/v%7B%7B%20pgrouting_release%20%7D%7D/pgrouting-%7B%7B%20pgrouting_release%20%7D%7D.tar.gz" dest: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz checksum: "{{ pgrouting_release_checksum }}" + timeout: 60 - name: pgRouting - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/03-pgtap.yml b/ansible/tasks/postgres-extensions/03-pgtap.yml index 6dc11f0e3..3ac2e988b 100644 --- a/ansible/tasks/postgres-extensions/03-pgtap.yml +++ b/ansible/tasks/postgres-extensions/03-pgtap.yml @@ -4,6 +4,7 @@ url: "https://github.com/theory/pgtap/archive/v%7B%7B%20pgtap_release%20%7D%7D.tar.gz" dest: /tmp/pgtap-{{ pgtap_release }}.tar.gz checksum: "{{ pgtap_release_checksum }}" + timeout: 60 - name: pgTAP - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/04-pg_cron.yml b/ansible/tasks/postgres-extensions/04-pg_cron.yml index 5bdf294f5..368d5190f 100644 --- a/ansible/tasks/postgres-extensions/04-pg_cron.yml +++ b/ansible/tasks/postgres-extensions/04-pg_cron.yml @@ -4,6 +4,7 @@ url: "https://github.com/citusdata/pg_cron/archive/refs/tags/v%7B%7B%20pg_cron_release%20%7D%7D.tar.gz" dest: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz checksum: "{{ pg_cron_release_checksum }}" + timeout: 60 - name: pg_cron - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/05-pgaudit.yml b/ansible/tasks/postgres-extensions/05-pgaudit.yml index 6d3b2bca7..743e94b05 100644 --- a/ansible/tasks/postgres-extensions/05-pgaudit.yml +++ b/ansible/tasks/postgres-extensions/05-pgaudit.yml @@ -12,6 +12,7 @@ url: "https://github.com/pgaudit/pgaudit/archive/refs/tags/%7B%7B%20pgaudit_release%20%7D%7D.tar.gz" dest: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz checksum: "{{ pgaudit_release_checksum }}" + timeout: 60 - name: pgAudit - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml index 6fd5cf9aa..9f2277674 100644 --- a/ansible/tasks/postgres-extensions/07-pgsql-http.yml +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -17,6 +17,7 @@ url: "https://github.com/pramsey/pgsql-http/archive/refs/tags/v%7B%7B%20pgsql_http_release%20%7D%7D.tar.gz" dest: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz checksum: "{{ pgsql_http_release_checksum }}" + timeout: 60 - name: pgsql-http - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml index 16fb5aa7e..2deb428e6 100644 --- a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml +++ b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml @@ -11,6 +11,7 @@ url: "https://github.com/okbob/plpgsql_check/archive/refs/tags/v%7B%7B%20plpgsql_check_release%20%7D%7D.tar.gz" dest: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz checksum: "{{ plpgsql_check_release_checksum }}" + timeout: 60 - name: plpgsql_check - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml index e27cfd9dc..606e36e26 100644 --- a/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml +++ b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml @@ -4,6 +4,7 @@ url: "https://github.com/eradman/pg-safeupdate/archive/refs/tags/%7B%7B%20pg_safeupdate_release%20%7D%7D.tar.gz" dest: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz checksum: "{{ pg_safeupdate_release_checksum }}" + timeout: 60 - name: pg-safeupdate - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/11-wal2json.yml b/ansible/tasks/postgres-extensions/11-wal2json.yml index 8fabbdd09..38c02f8f8 100644 --- a/ansible/tasks/postgres-extensions/11-wal2json.yml +++ b/ansible/tasks/postgres-extensions/11-wal2json.yml @@ -4,6 +4,7 @@ url: "https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_%7B%7B%20wal2json_release%20%7D%7D.tar.gz" dest: /tmp/wal2json-{{ wal2json_release }}.tar.gz checksum: "{{ wal2json_release_checksum }}" + timeout: 60 - name: wal2json - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml index 8330728cf..3edfb6aab 100644 --- a/ansible/tasks/postgres-extensions/12-pljava.yml +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -14,6 +14,7 @@ url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz dest: /tmp/pljava-{{ pljava_release }}.tar.gz checksum: "{{ pljava_release_checksum }}" + timeout: 60 - name: pljava - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/15-pg_net.yml b/ansible/tasks/postgres-extensions/15-pg_net.yml index 8f00f8508..d8b39b5bd 100644 --- a/ansible/tasks/postgres-extensions/15-pg_net.yml +++ b/ansible/tasks/postgres-extensions/15-pg_net.yml @@ -11,6 +11,7 @@ url: "https://github.com/supabase/pg_net/archive/refs/tags/v%7B%7Bpg_net_release%7D%7D.tar.gz" dest: /tmp/pg_net-{{ pg_net_release }}.tar.gz checksum: "{{ pg_net_release_checksum }}" + timeout: 60 - name: pg_net - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/16-rum.yml b/ansible/tasks/postgres-extensions/16-rum.yml index 9a2fad7ac..2284ff1cd 100644 --- a/ansible/tasks/postgres-extensions/16-rum.yml +++ b/ansible/tasks/postgres-extensions/16-rum.yml @@ -4,6 +4,7 @@ url: "https://github.com/postgrespro/rum/archive/refs/tags/%7B%7Brum_release%7D%7D.tar.gz" dest: /tmp/rum-{{ rum_release }}.tar.gz checksum: "{{ rum_release_checksum }}" + timeout: 60 - name: rum - unpack archive unarchive: diff --git a/ansible/tasks/postgres-extensions/18-pgsodium.yml b/ansible/tasks/postgres-extensions/18-pgsodium.yml index 727fd7283..cb6e11148 100644 --- a/ansible/tasks/postgres-extensions/18-pgsodium.yml +++ b/ansible/tasks/postgres-extensions/18-pgsodium.yml @@ -4,6 +4,7 @@ url: "https://download.libsodium.org/libsodium/releases/libsodium-%7B%7B%20libsodium_release%20%7D%7D.tar.gz" dest: /tmp/libsodium-{{ libsodium_release }}.tar.gz checksum: "{{ libsodium_release_checksum }}" + timeout: 60 - name: libsodium - unpack archive unarchive: @@ -34,6 +35,7 @@ url: "https://github.com/michelp/pgsodium/archive/refs/tags/v1.3.0.tar.gz" dest: /tmp/pgsodium-{{ pgsodium_release }}.tar.gz checksum: "{{ pgsodium_release_checksum }}" + timeout: 60 - name: pgsodium - unpack archive unarchive: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 1bbcaeb0c..45368659c 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -42,6 +42,7 @@ url: https://ftp.postgresql.org/pub/source/v{{ postgresql_release }}/postgresql-{{ postgresql_release }}.tar.gz dest: /tmp checksum: "{{ postgresql_release_checksum }}" + timeout: 60 - name: Postgres - unpack archive unarchive: diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index cfd0647a1..c4c60c855 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -13,6 +13,7 @@ url: "https://github.com/PostgREST/postgrest/releases/download/v%7B%7B%20postgrest_release%20%7D%7D/postgrest-v%7B%7B%20postgrest_release%20%7D%7D-linux-x64-static.tar.xz" dest: /tmp/postgrest.tar.xz checksum: "{{ postgrest_arm_release_checksum }}" + timeout: 60 when: platform == "arm64" - name: PostgREST - download ubuntu binary archive (x86) @@ -20,6 +21,7 @@ url: "https://github.com/PostgREST/postgrest/releases/download/v%7B%7B%20postgrest_release%20%7D%7D/postgrest-v%7B%7B%20postgrest_release%20%7D%7D-ubuntu-aarch64.tar.xz" dest: /tmp/postgrest.tar.xz checksum: "{{ postgrest_x86_release_checksum }}" + timeout: 60 when: platform == "amd64" - name: PostgREST - unpack archive in /opt diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index b329a6119..fedf5b220 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -9,12 +9,14 @@ get_url: url: "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-%7B%7B%20aws_cli_release%20%7D%7D.zip" dest: "/tmp/awscliv2.zip" + timeout: 60 when: platform == "arm64" - name: AWS CLI (x86) get_url: url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-%7B%7B%20aws_cli_release%20%7D%7D.zip" dest: "/tmp/awscliv2.zip" + timeout: 60 when: platform == "amd64" - name: AWS CLI - expand diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 4e1c71242..acbfd23e1 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -13,6 +13,7 @@ url: "https://golang.org/dl/go%7B%7B%20golang_version%20%7D%7D.linux-%7B%7B%20platform%20%7D%7D.tar.gz" dest: /tmp checksum: "{{ golang_version_checksum[platform] }}" + timeout: 60 - name: unpack go archive unarchive: From 3c0201ad6d3586ad16a81abe9b8341bbd5d4edc7 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 25 Nov 2021 13:43:36 +0800 Subject: [PATCH 27/29] further hardening of pgbouncer - no access to public schema - cannot connect to port 5432 by pgbouncer user other than localhost --- ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql | 2 ++ ansible/tasks/setup-pgbouncer.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql b/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql index bc1342f8d..c10ce44fd 100644 --- a/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql +++ b/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql @@ -1,5 +1,7 @@ CREATE USER pgbouncer; +REVOKE ALL PRIVILEGES ON SCHEMA public FROM pgbouncer; + CREATE SCHEMA pgbouncer AUTHORIZATION pgbouncer; CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT) diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 3e805cc8d..bc7d333d8 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -15,6 +15,7 @@ url: "https://www.pgbouncer.org/downloads/files/%7B%7B%20pgbouncer_release%20%7D%7D/pgbouncer-%7B%7B%20pgbouncer_release%20%7D%7D.tar.gz" dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz checksum: "{{ pgbouncer_release_checksum }}" + timeout: 60 - name: PgBouncer - unpack archive unarchive: @@ -84,8 +85,9 @@ insertafter: '# Default:' line: "{{ item }}" with_items: + - "host all pgbouncer 0.0.0.0/0 reject" - "host all pgbouncer 127.0.0.1/32 scram-sha-256" - - "# Allow connection by pgbouncer user" + - "# Connection configuration for pgbouncer user" # Run PgBouncer SQL script - name: Transfer init SQL files From 8584437cd7d7aebad58dcbd16f0c766f3f536475 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Thu, 25 Nov 2021 13:55:48 +0800 Subject: [PATCH 28/29] bump common.vars.json --- common.vars.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.vars.json b/common.vars.json index 83454511d..a0386e6e7 100644 --- a/common.vars.json +++ b/common.vars.json @@ -1,3 +1,3 @@ { - "postgres-version": "13.3.0.5-rc0" + "postgres-version": "14.1.0.4-rc0" } From fcb97273ceebe9cd342ee0ce8d8795babaac8726 Mon Sep 17 00:00:00 2001 From: dragarcia Date: Mon, 29 Nov 2021 14:12:03 +0800 Subject: [PATCH 29/29] Enhancements to docker images (#108) * GA for dockerhub * Shift removal of unused packages at the bottom - For some reason, this disrupts the ansible process - Err: the connection plugin 'local' was not found * No longer use init.sh for docker * Use Dockerfile when building docker images * cleanup /tmp via the Dockerfile * more changes to docker build - move more cleanup to the Dockerfile - run everything under one command in the Dockerfile * Commenting these out for now - Need to configure runners first - Will uncomment in another PR once done * ignore caretion of symbolic link for plv8 in docker arm build * Temporarily remove CI files for Dockerhub --- Dockerfile | 21 ++++++++ ansible/files/docker_mnt/init.sh | 3 -- ansible/playbook-docker.yml | 40 +-------------- ansible/tasks/docker/cleanup.yml | 51 +------------------ ansible/tasks/docker/setup.yml | 31 ++++------- ansible/tasks/postgres-extensions/13-plv8.yml | 1 + 6 files changed, 36 insertions(+), 111 deletions(-) create mode 100644 Dockerfile delete mode 100644 ansible/files/docker_mnt/init.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..110f70aaf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +ARG PLATFORM +ARG VERSION + +FROM --platform=$PLATFORM postgres:$VERSION + +COPY ansible/ /tmp/ansible/ + +RUN apt update && \ + apt install -y ansible && \ + cd /tmp/ansible && \ + ansible-playbook playbook-docker.yml && \ + apt -y update && \ + apt -y upgrade && \ + apt -y autoremove && \ + apt -y autoclean && \ + apt install -y default-jdk-headless && \ + rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* + +ENV LANGUAGE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 \ No newline at end of file diff --git a/ansible/files/docker_mnt/init.sh b/ansible/files/docker_mnt/init.sh deleted file mode 100644 index fd12bbe4a..000000000 --- a/ansible/files/docker_mnt/init.sh +++ /dev/null @@ -1,3 +0,0 @@ -cat /etc/postgresql/postgresql.conf > $PGDATA/postgresql.conf -echo "host replication $POSTGRES_USER 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf -echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf \ No newline at end of file diff --git a/ansible/playbook-docker.yml b/ansible/playbook-docker.yml index ea699339f..abeb15428 100644 --- a/ansible/playbook-docker.yml +++ b/ansible/playbook-docker.yml @@ -1,26 +1,5 @@ -- name: Preparing Docker container - hosts: localhost - tasks: - - name: Pull Postgres Image - docker_container: - name: "supabase-postgres-build" - image: "postgres:13.3" - env: - LANGUAGE: "en_US.UTF-8" - LANG: "en_US.UTF-8" - LC_ALL: "en_US.UTF-8" - state: started - memory: 4G - memory_swap: 6G - command: tail -f /dev/null - - name: Add Postgres Image to Ansible Hosts - add_host: - name: "supabase-postgres-build" - ansible_connection: docker - ansible_ssh_user: root - - name: Build Supabase Postgres - hosts: "supabase-postgres-build" + hosts: localhost gather_facts: false vars_files: @@ -34,19 +13,4 @@ import_tasks: tasks/setup-extensions.yml - name: Cleanup container - import_tasks: tasks/docker/cleanup.yml - -- name: Create supabase/postgres docker image - hosts: localhost - tasks: - - name: Commit Docker image - command: docker commit --change='CMD ["postgres"]' "supabase-postgres-build" "supabase/postgres" - -- name: Clean Up Postgres Image - hosts: localhost - tasks: - - name: Remove Running Base Image - docker_container: - name: supabase-postgres-build - state: absent - force_kill: yes \ No newline at end of file + import_tasks: tasks/docker/cleanup.yml \ No newline at end of file diff --git a/ansible/tasks/docker/cleanup.yml b/ansible/tasks/docker/cleanup.yml index 4552d8030..177a5f847 100644 --- a/ansible/tasks/docker/cleanup.yml +++ b/ansible/tasks/docker/cleanup.yml @@ -1,5 +1,3 @@ - - - name: Cleanup - remove build dependencies apt: pkg: @@ -17,51 +15,4 @@ - cmake - ninja-build - python - state: absent - -- name: Cleanup - apt update and apt upgrade - apt: update_cache=yes upgrade=yes - # SEE http://archive.vn/DKJjs#parameter-upgrade - -- name: Cleanup - remove dependencies that are no longer required - apt: - autoremove: yes - -- name: Cleanup - remove useless packages from the cache - apt: - autoclean: yes - -- name: Cleanup - reinstall headless jdk - apt: - pkg: - - default-jdk-headless - update_cache: yes - install_recommends: no - -- name: Cleanup - find all files in /tmp - find: - paths: /tmp - file_type: any - register: tmp_items_to_delete - -- name: Cleanup - delete all items in /tmp - file: - path: "/tmp/{{ item.path | basename }}" - state: absent - force: yes - with_items: "{{ tmp_items_to_delete.files }}" - -- name: Cleanup - find all files in /var/lib/apt/lists/* - find: - paths: /var/lib/apt/lists - file_type: any - register: var_items_to_delete - -- name: Cleanup - delete all items in /tmp - file: - path: "/var/lib/apt/lists/{{ item.path | basename }}" - state: absent - force: yes - with_items: "{{ var_items_to_delete.files }}" - - \ No newline at end of file + state: absent \ No newline at end of file diff --git a/ansible/tasks/docker/setup.yml b/ansible/tasks/docker/setup.yml index d669f8870..433680285 100644 --- a/ansible/tasks/docker/setup.yml +++ b/ansible/tasks/docker/setup.yml @@ -30,7 +30,17 @@ - name: Setup - import postgresql.conf synchronize: src: files/postgresql_config/postgresql.conf.j2 - dest: etc/postgresql/postgresql.conf + dest: /etc/postgresql/postgresql.conf + +- name: Setup - import postgresql.conf + synchronize: + src: files/postgresql_config/pg_hba.conf.j2 + dest: /etc/postgresql/pg_hba.conf + +- name: Setup - import postgresql.conf + synchronize: + src: files/postgresql_config/pg_ident.conf.j2 + dest: /etc/postgresql/pg_ident.conf - set_fact: regex_string: "#unix_socket_directories = '/tmp'" @@ -48,22 +58,3 @@ path: /etc/postgresql/postgresql.conf regexp: '{{ regex_string }}' replace: unix_socket_directories = '/var/run/postgresql' - -- name: Setup - modify hba_file directory - become: yes - replace: - path: /etc/postgresql/postgresql.conf - regexp: hba_file = '/etc/postgresql/pg_hba.conf' - replace: hba_file = '/var/lib/postgresql/data/pg_hba.conf' - -- name: Setup - modify ident_file directory - become: yes - replace: - path: /etc/postgresql/postgresql.conf - regexp: ident_file = '/etc/postgresql/pg_ident.conf' - replace: ident_file = '/var/lib/postgresql/data/pg_ident.conf' - -- name: Setup - add init script to /docker-entrypoint-initdb.d - synchronize: - src: files/docker_mnt/init.sh - dest: /docker-entrypoint-initdb.d/init.sh \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/13-plv8.yml b/ansible/tasks/postgres-extensions/13-plv8.yml index 36c11a449..48e8830f0 100644 --- a/ansible/tasks/postgres-extensions/13-plv8.yml +++ b/ansible/tasks/postgres-extensions/13-plv8.yml @@ -35,6 +35,7 @@ dest: /lib/aarch64-linux-gnu/libc++.so state: link when: platform == "arm64" + ignore_errors: yes # not needed for docker build - name: plv8 - build make: