enable tests for the physical basebackup case

Author: agneum

.gitlab-ci.yml

@@ -328,6 +328,7 @@ build-image-swagger-latest:
   script:
     - bash test/1.synthetic.sh
     - bash test/2.logical_generic.sh
+    - bash test/4.physical_basebackup.sh
   after_script:
     - bash test/_cleanup.sh
   tags:

test/4.physical_basebackup.sh

@@ -3,53 +3,69 @@ set -euxo pipefail
 
 TAG="${TAG:-"master"}"
 IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
+DLE_SERVER_NAME="dblab_server_test"
 
 # Environment variables for replacement rules
 export SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
 export SOURCE_PORT="${SOURCE_PORT:-7432}"
 export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
 export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
 export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
+export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+export DLE_TEST_POOL_NAME="test_dblab_pool"
+export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
+export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
+export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
 
 DIR=${0%/*}
 
 if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
 ### Step 0. Create source database
-  sudo rm -rf "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test || true
+  TMP_DATA_DIR="/tmp/dle_test/physical_basebackup"
+  cleanup_testdata_dir() {
+    sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
+  }
+
+  trap cleanup_testdata_dir EXIT
+
+  cleanup_testdata_dir
+  sudo docker rm postgres"${POSTGRES_VERSION}" || true
+
   sudo docker run \
     --name postgres"${POSTGRES_VERSION}" \
     --label pgdb \
+    --label dblab_test \
     --privileged \
     --publish 172.17.0.1:"${SOURCE_PORT}":5432 \
     --env PGDATA=/var/lib/postgresql/pgdata \
     --env POSTGRES_USER="${SOURCE_USERNAME}" \
     --env POSTGRES_PASSWORD="${SOURCE_PASSWORD}" \
     --env POSTGRES_DB=test \
     --env POSTGRES_HOST_AUTH_METHOD=md5 \
-    --volume "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
+    --volume "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
     --detach \
     postgres:"${POSTGRES_VERSION}-alpine"
 
   for i in {1..300}; do
-    sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -d test -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -d test -U postgres -c 'select' > /dev/null 2>&1  && break || echo "test database is not ready yet"
     sleep 1
   done
 
   # add "host replication" to pg_hba.conf
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" bash -c 'echo "host replication all 0.0.0.0/0 md5" >> $PGDATA/pg_hba.conf'
+  sudo docker exec postgres"${POSTGRES_VERSION}" bash -c 'echo "host replication all 0.0.0.0/0 md5" >> $PGDATA/pg_hba.conf'
   # reload conf
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -U postgres -c 'select pg_reload_conf()'
+  sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c 'select pg_reload_conf()'
 
   # Generate data in the test database using pgbench
   # 1,000,000 accounts, ~0.14 GiB of data.
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
+  sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
 
   # Database info
-  sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
+  sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
 fi
 
 ### Step 1. Prepare a machine with disk, Docker, and ZFS
-source "${DIR}/_prerequisites.ubuntu.sh"
+#source "${DIR}/_prerequisites.ubuntu.sh"
 source "${DIR}/_zfs.file.sh"
 
 
@@ -69,6 +85,10 @@ yq eval -i '
   .global.debug = true |
   .global.telemetry.enabled = false |
   .localUI.enabled = false |
+  .server.port = env(DLE_SERVER_PORT) |
+  .poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
+  .provision.portPool.from = env(DLE_PORT_POOL_FROM) |
+  .provision.portPool.to = env(DLE_PORT_POOL_TO) |
   .databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
   .retrieval.spec.physicalRestore.options.envs.PGUSER = strenv(SOURCE_USERNAME) |
   .retrieval.spec.physicalRestore.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) |
@@ -77,17 +97,19 @@ yq eval -i '
   .retrieval.spec.physicalSnapshot.options.envs.PGUSER = strenv(SOURCE_USERNAME) |
   .retrieval.spec.physicalSnapshot.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) |
   .retrieval.spec.physicalSnapshot.options.envs.PGHOST = strenv(SOURCE_HOST) |
-  .retrieval.spec.physicalSnapshot.options.envs.PGPORT = env(SOURCE_PORT)
+  .retrieval.spec.physicalSnapshot.options.envs.PGPORT = env(SOURCE_PORT) |
+  .retrieval.spec.physicalRestore.options.customTool.command = "pg_basebackup -X stream -D " + strenv(DLE_TEST_MOUNT_DIR) + "/" + strenv(DLE_TEST_POOL_NAME) + "/data"
 ' "${configDir}/server.yml"
 
 ## Launch Database Lab server
 sudo docker run \
-  --name dblab_server \
+  --name ${DLE_SERVER_NAME} \
   --label dblab_control \
+  --label dblab_test \
   --privileged \
-  --publish 2345:2345 \
+  --publish ${DLE_SERVER_PORT}:${DLE_SERVER_PORT} \
   --volume /var/run/docker.sock:/var/run/docker.sock \
-  --volume /var/lib/dblab:/var/lib/dblab/:rshared \
+  --volume ${DLE_TEST_MOUNT_DIR}:${DLE_TEST_MOUNT_DIR}/:rshared \
   --volume "${configDir}":/home/dblab/configs:ro \
   --volume "${metaDir}":/home/dblab/meta \
   --volume /sys/kernel/debug:/sys/kernel/debug:rw \
@@ -98,11 +120,11 @@ sudo docker run \
   "${IMAGE2TEST}"
 
 # Check the Database Lab Engine logs
-sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
+sudo docker logs ${DLE_SERVER_NAME} -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
 
 ### Waiting for the Database Lab Engine initialization.
 for i in {1..30}; do
-  curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
+  curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
   sleep 10
 done
 
@@ -118,7 +140,7 @@ dblab --version
 # Initialize CLI configuration
 dblab init \
   --environment-id=test \
-  --url=http://localhost:2345 \
+  --url=http://localhost:${DLE_SERVER_PORT} \
   --token=secret_token \
   --insecure
 
@@ -134,14 +156,14 @@ dblab clone create \
 
 # Connect to a clone and check the available table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 # Drop table
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
 
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 ## Reset clone
 dblab clone reset testclone
@@ -151,11 +173,14 @@ dblab clone status testclone
 
 # Check the database objects (everything should be the same as when we started)
 PGPASSWORD=secret_password psql \
-  "host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
+  "host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
 
 ### Step 4. Destroy clone
 dblab clone destroy testclone
 dblab clone list
 
+## Stop DLE.
+sudo docker stop ${DLE_SERVER_NAME}
+
 ### Finish. clean up
 source "${DIR}/_cleanup.sh"