diff --git a/.gitignore b/.gitignore index 064695e..242deb0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ example.com.key postgres.example.com.csr test.pem secret.tfvars +ubuntu.pem diff --git a/README.md b/README.md index 9069447..6bfd419 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ -[[_TOC_]] +# Database Lab Terraform Module -# How to setup Database Lab using Terraform in AWS - -This [Terraform Module](https://www.terraform.io/docs/language/modules/index.html) is responsible for deploying the [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab) to cloud hosting providers. +This [Terraform Module](https://www.terraform.io/docs/language/modules/index.html) can be used as a template for deploying the [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab) to cloud hosting providers. Please feel free to tailor it to meet your requirements. Your source PostgreSQL database can be located anywhere, but DLE with other components will be created on an EC2 instance under your AWS account. Currently, only "logical" mode of data retrieval (dump/restore) is supported – the only available method for managed PostgreSQL cloud services such as RDS Postgres, RDS Aurora Postgres, Azure Postgres, or Heroku. "Physical" mode is not yet supported, but it will be in the future. More about various data retrieval options for DLE: https://postgres.ai/docs/how-to-guides/administration/data. -## Supported Cloud Platforms: +## Supported Cloud Platforms - AWS ## Prerequisites @@ -14,14 +12,14 @@ Your source PostgreSQL database can be located anywhere, but DLE with other comp - [Terraform Installed](https://learn.hashicorp.com/tutorials/terraform/install-cli) (minimal version: 1.0.0) - AWS [Route 53](https://aws.amazon.com/route53/) Hosted Zone (For setting up TLS) for a domain or sub-domain you control - You must have AWS Access Keys and a default region in your Terraform environment (See section on required IAM Permissions) -- The DLE runs on an EC2 instance which can be accessed using a selected set of SSH keys uploaded to EC2. Use the Terraform parameter `aws_keypair` to specify which EC2 Keypair to use +- The DLE runs on an EC2 instance which can be accessed using a selected set of SSH keys uploaded to EC2. - Required IAM Permissions: to successfully run this Terraform module, the IAM User/Role must have the following permissions: * Read/Write permissions on EC2 * Read/Write permissions on Route53 * Read/Write permissions on Cloudwatch -## Configuration overview -- :construction: Currently, it is supposed that you run `terraform` commands on a Linux machine. MacOS and Windows support is not yet implemented (but planned). +## How to use +- :construction: Currently, it is supposed that you run `terraform` commands on a Linux machine or MacOS. Windows support is not yet implemented (but planned). - It is recommended to clone this Git repository and adjust for your needs. Below we provide the detailed step-by-step instructions for quick start (see "Quick start") for a PoC setup - To configure parameters used by Terraform (and the Database Lab Engine itself), you will need to modify `terraform.tfvars` and create a file with secrets (`secret.tfvars`) - This Terraform module can be run independently or combined with any other standard Terraform module. You can learn more about using Terraform and the Terraform CLI [here](https://www.terraform.io/docs/cli/commands/index.html) @@ -31,7 +29,7 @@ Your source PostgreSQL database can be located anywhere, but DLE with other comp - values passed on the command line - All variables starting with `postgres_` represent the source database connection information for the data (from that database) to be fetched by the DLE. That database must be accessible from the instance hosting the DLE (that one created by Terraform) -## How-to guide: using this Terraform module to set up DLE and its components +## Quick start The following steps were tested on Ubuntu 20.04 but supposed to be valid for other Linux distributions without significant modification. 1. SSH to any machine with internet access, it will be used as deployment machine @@ -51,53 +49,45 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth ``` 1. Edit `terraform.tfvars` file. In our example, we will use Heroku demo database as a source: ```config - dle_version_full = "2.4.1" + dle_version_full = "2.5.0" aws_ami_name = "DBLABserver*" - aws_keypair = "YOUR_AWS_KEYPAIR" aws_deploy_region = "us-east-1" aws_deploy_ebs_availability_zone = "us-east-1a" - aws_deploy_ec2_instance_type = "t2.large" + aws_deploy_ec2_instance_type = "c5.large" aws_deploy_ec2_instance_tag_name = "DBLABserver-ec2instance" - aws_deploy_ebs_size = "40" + aws_deploy_ebs_size = "10" + aws_deploy_ec2_volumes_count = "2" aws_deploy_ebs_type = "gp2" aws_deploy_allow_ssh_from_cidrs = ["0.0.0.0/0"] aws_deploy_dns_api_subdomain = "tf-test" # subdomain in aws.postgres.ai, fqdn will be ${dns_api_subdomain}-engine.aws.postgres - # Source – two options. Choose one of two: - # - direct connection to source DB - # - dump stored on AWS S3 - - # option 1 – direct PG connection - source_type = "postgres" # source is working dome postgres database source_postgres_version = "13" - source_postgres_host = "ec2-3-215-57-87.compute-1.amazonaws.com" # an example DB at Heroku + source_postgres_host = "ec2-3-215-57-87.compute-1.amazonaws.com" source_postgres_port = "5432" - source_postgres_dbname = "d3dljqkrnopdvg" # an example DB at Heroku - source_postgres_username = "bfxuriuhcfpftt" # an example DB at Heroku - - # option 2 – dump on S3. Important: your AWS user has to be able to create IAM roles to work with S3 buckets in your AWS account - # source_type = 's3' # source is dump stored on demo s3 bucket - # source_pgdump_s3_bucket = "tf-demo-dump" # This is an example public bucket - # source_pgdump_path_on_s3_bucket = "heroku.dmp" # This is an example dump from demo database - + source_postgres_dbname = "d3dljqkrnopdvg" # this is an existing DB (Heroku example DB) + source_postgres_username = "bfxuriuhcfpftt" # in secret.tfvars, use: source_postgres_password = "dfe01cbd809a71efbaecafec5311a36b439460ace161627e5973e278dfe960b7" dle_debug_mode = "true" dle_retrieval_refresh_timetable = "0 0 * * 0" postgres_config_shared_preload_libraries = "pg_stat_statements,logerrors" # DB Migration Checker requires logerrors extension platform_project_name = "aws_test_tf" + + # list of ssh public keys stored in files + ssh_public_keys_files_list = ["~/.ssh/id_rsa.pub"] + # or provided inline + ssh_public_keys_list = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhbblazDXCFEc21DtFzprWC8DiqidnVRROzp6J6BeJR9+XydPUtl0Rt2mcNvxL5ro5bI9u5JRW8aDd6s+Orpr66hEDdwQTbT1wp5nyduFQcT3rR +aeDSilQvAHjr4/z/GZ6IgZ5MICSIh5hJJagHoxAVqeS9dCA27tv/n2T2XrxIUeBhywH1EmfwrnEw97tHM8F+yegayFDI1nVOUWUIxFMaygMygix8uKbQ2fl4rkkxG2oEx7uyAFMXHt4bewNbZuAp8b/b5ODL6tGHuHhcwfbWGriCO+l7UOf1K9maTx00o4wkzAPyd+qs70y/1iMX2YOOLYaYYdptEnFal2DVoD example@example.com" + ] ``` 1. Create `secret.tfvars` containing `source_postgres_password`, `platform_access_token`, and `vcs_github_secret_token`. An example: ```config - source_postgres_password = "dfe01cbd809a71efbaecafec5311a36b439460ace161627e5973e278dfe960b7" # an example DB at Heroku + source_postgres_password = "YOUR_DB_PASSWORD" # todo: put pwd for heroku example DB here platform_access_token = "YOUR_ACCESS_TOKEN" # to generate, open https://console.postgres.ai/, choose your organization, # then "Access tokens" in the left menu - vcs_github_secret_token = "vcs_secret_token" # generate a personal access token with scope of "repo" + vcs_github_secret_token = "vcs_secret_token" # to generate, open https://github.com/settings/tokens/new ``` - To generate a personal GitHub access token with the scope of "repo", open the [guide on GitHub Docs](https://docs.github.com/en/github/authenticating-to-github/keeping-your-account-and-data-secure/creating-a-personal-access-token) and follow the instructions. - - Note that the "repo" scope essentially gives full access to all user-controlled repositories. Should you have any concerns about which repositories the DLE can have access to, consider using a separate GitHub account that has access to the reduced number of repositories. 1. Initialize ```shell terraform init @@ -108,21 +98,32 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth export AWS_SECRET_ACCESS_KEY = "accesskey" ``` 1. Deploy: - ```shell - terraform apply -var-file="secret.tfvars" -auto-approve + ``` + terraform apply -var-file="secret.tfvars" -auto-approve && terraform output -raw next_steps ``` 1. If everything goes well, you should get an output like this: ```config - vcs_db_migration_checker_verification_token = "gsio7KmgaxECfJ80kUx2tUeIf4kEXZex" - dle_verification_token = "zXPodd13LyQaKgVXGmSCeB8TUtnGNnIa" - ec2_public_dns = "ec2-11-111-111-11.us-east-2.compute.amazonaws.com" - ec2instance = "i-0000000000000" - ip = "11.111.111.11" - platform_joe_signing_secret = "lG23qZbUh2kq0ULIBfW6TRwKzqGZu1aP" - public_dns_name = "demo-api-engine.aws.postgres.ai" # todo: this should be URL, not hostname – further we'll need URL, with protocol – `https://` - ``` -1. To verify result and check the progress, you might want to connect to the just-created EC2 machine using IP address or hostname from the Terraform output. In our example, it can be done using this one-liner (you can find more about DLE logs and configuration on this page: https://postgres.ai/docs/how-to-guides/administration/engine-manage): + ##################################################################### + + Congratulations! Database Lab Engine installed. + Data initialization may take time, depending on the database size. + + You should be able to work with all DLE interfaces already: + - [RECOMMENDED] UI: https://tf-test.aws.postgres.ai:446 + - CLI: dblab init --url=https://tf-test.aws.postgres.ai --token=sDTPu17pzXhW9DkhcSGpAMj72KgiIJxG --environment="i-0687b060f45314be5" --insecure + - API: https://tf-test.aws.postgres.ai + - SSH connection for troubleshooting: ssh ubuntu@3.92.133.178 -i dmitry-DBLABserver-ec2instance.pem + + (Use verification token: sDTPu17pzXhW9DkhcSGpAMj72KgiIJxG) + + For support, go to https://postgres.ai/contact. + + ##################################################################### + + ``` + +1. To verify result and check the progress, you might want to connect to the just-created EC2 machine using IP address or hostname from the Terraform output and ssh key from ssh_public_keys_files_list and/or ssh_public_keys_list variables. In our example, it can be done using this one-liner (you can find more about DLE logs and configuration on this page: https://postgres.ai/docs/how-to-guides/administration/engine-manage): ```shell echo "sudo docker logs dblab_server -f" | ssh ubuntu@18.118.126.25 -i postgres_ext_test.pem ``` @@ -183,10 +184,14 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth } ``` +This is it! + +If you need to remove everything created by this Terraform module, you can run `terraform destroy -var-file="secret.tfvars" -auto-approve`. Do not forget to do it if you're just experimenting. Otherwise, if you leave infrastructure blocks running, they might significantly affect your AWS bill (depending on the EC2 instance family you've chosen, the disk type, and size). + ## Important Note When the DLE creates new database clones, it makes them available on incremental ports in the 6000 range (e.g. 6000, 6001, ...). The DLE CLI will also report that the clone is available on a port in the 6000 range. However, please note that these are the ports when accessing the DLE from `localhost`. This Terraform module deploys [Envoy](https://www.envoyproxy.io/) to handle SSL termination and port forwarding to connect to DLE generated clones. -Bottom Line: When connecting to clones, add `3000` to the port number reported by the DLE CLI to connect to the clone. for example, if the CLI reports that a new clone is available at port `6001` connect that clone at port `9001`. +Follow the [how-to guide](https://postgres.ai/docs/how-to-guides/administration/install-database-lab-with-terraform) to install Database Lab with Terraform on AWS ## Known Issues ### Certificate Authority Authorization (CAA) for your Hosted Zone diff --git a/dle-logical-init.sh.tpl b/dle-logical-init.sh.tpl index d873b26..0095e39 100644 --- a/dle-logical-init.sh.tpl +++ b/dle-logical-init.sh.tpl @@ -4,10 +4,10 @@ set -x sleep 20 #run certbot and copy files to envoy -# to avoid restrinctions from letsencrypt like "There were too many requests of a given type :: +# to avoid restrictions from letsencrypt like "There were too many requests of a given type :: # Error creating new order :: too many certificates (5) already issued for this exact set of domains # in the last 168 hours: demo-api-engine.aws.postgres.ai: see https://letsencrypt.org/docs/rate-limits/" -# follwing three lines were commented out and mocked up. In real implementation inline certs have to be +# following three lines were commented out and mocked up. In real implementation inline certs have to be # removed and letsencrypt generated certs should be used @@ -86,75 +86,128 @@ EOF sudo systemctl enable envoy sudo systemctl start envoy -#create zfs pools -disks=(${dle_disks}) -for i in $${!disks[@]}; do +# create zfs pools +# Get the full list of disks available and then make attempts +# to create zpool on each. Here we assume that the system disk +# will be skipped because it already has a filesystem. +# This is a "brute force" approach that we probably want to +# rework, but for now we leave it as is because it seems that +# `/dev/../by-id` doesn't really work for all EC2 types. + +disks=$(lsblk -ndp -e7 --output NAME) # TODO: this is not needed, used now for debug only + +i=1 + +sleep 10 # Not elegant at all, we need a better way to wait till the moment when all disks are available + +# Show all disks in alphabetic order; "-e7" to exclude loop devices +for disk in $disks; do sudo zpool create -f \ - -O compression=on \ - -O atime=off \ - -O recordsize=128k \ - -O logbias=throughput \ - -m /var/lib/dblab/dblab_pool_0$i\ - dblab_pool_0$i \ - $${disks[$i]} + -O compression=on \ + -O atime=off \ + -O recordsize=128k \ + -O logbias=throughput \ + -m /var/lib/dblab/dblab_pool_$(printf "%02d" $i)\ + dblab_pool_$(printf "%02d" $i) \ + $disk \ + && ((i=i+1)) # increment if succeeded done # Adjust DLE config -mkdir ~/.dblab -curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/config.example.logical_generic.yml --output ~/.dblab/server.yml -sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/server.yml -sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" ~/.dblab/server.yml -sed -ri "s/^(\s*)(timetable:.*$)/\1timetable: \"${dle_retrieval_refresh_timetable}\"/" ~/.dblab/server.yml -sed -ri "s/^(\s*)(forceInit:.*$)/\1forceInit: true/" ~/.dblab/server.yml -sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" ~/.dblab/server.yml +dle_config_path="/home/ubuntu/.dblab/engine/configs" +dle_meta_path="/home/ubuntu/.dblab/engine/meta" +postgres_conf_path="/home/ubuntu/.dblab/postgres_conf" + +mkdir -p $dle_config_path +mkdir -p $dle_meta_path +mkdir -p $postgres_conf_path + +curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/config.example.logical_generic.yml --output $dle_config_path/server.yml +curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/standard/postgres/control/pg_hba.conf \ + --output $postgres_conf_path/pg_hba.conf +curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/standard/postgres/control/postgresql.conf --output $postgres_conf_path/postgresql.conf +cat /tmp/postgresql_clones_custom.conf >> $postgres_conf_path/postgresql.conf + +yq e -i ' + .global.debug=${dle_debug_mode} | + .server.verificationToken="${dle_verification_token}" | + .retrieval.refresh.timetable="${dle_retrieval_refresh_timetable}" | + .retrieval.spec.logicalDump.options.source.connection.dbname="${source_postgres_dbname}" | + .retrieval.spec.logicalRestore.options.forceInit=true | + .databaseContainer.dockerImage="postgresai/extended-postgres:${source_postgres_version}" +' $dle_config_path/server.yml + # Enable Platform -sed -ri "s/^(\s*)(#platform:$)/\1platform: /" ~/.dblab/server.yml -sed -ri "s/^(\s*)(# url: \"https\\:\\/\\/postgres.ai\\/api\\/general\"$)/\1 url: \"https\\:\\/\\/postgres.ai\\/api\\/general\" /" ~/.dblab/server.yml -sed -ri "s/^(\s*)(# accessToken: \"platform_access_token\"$)/\1 accessToken: \"${platform_access_token}\"/" ~/.dblab/server.yml -sed -ri "s/^(\s*)(# enablePersonalTokens: true$)/\1 enablePersonalTokens: true/" ~/.dblab/server.yml -sed -ri "s/:13/:${source_postgres_version}/g" ~/.dblab/server.yml +yq e -i ' + .platform.url = "https://postgres.ai/api/general" | + .platform.accessToken = "${platform_access_token}" | + .platform.enablePersonalTokens = true | +' $dle_config_path/server.yml case "${source_type}" in postgres) - sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" ~/.dblab/server.yml - sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" ~/.dblab/server.yml - sed -ri "s/^(\s*)( username: postgres$)/\1 username: ${source_postgres_username}/" ~/.dblab/server.yml - sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" ~/.dblab/server.yml - #restore pg_dump via pipe - without saving it on the disk - sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" ~/.dblab/server.yml - sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" ~/.dblab/server.yml - sed -ri "s/^(\s*)(# forceInit: false.*$)/\1 forceInit: true /" ~/.dblab/server.yml - sed -ri "s/^(\s*)( # configs:$)/\1 configs: /" ~/.dblab/server.yml - sed -ri "s/^(\s*)( # shared_preload_libraries: .*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml - sed -ri "s/^(\s*)( shared_preload_libraries:.*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml - sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" ~/.dblab/server.yml + # Mount directory to store dump files. + extra_mount="--volume /var/lib/dblab/dblab_pool_00/dump:/var/lib/dblab/dblab_pool/dump" + + yq e -i ' + .retrieval.spec.logicalDump.options.source.connection.host = "${source_postgres_host}" | + .retrieval.spec.logicalDump.options.source.connection.port = ${source_postgres_port} | + .retrieval.spec.logicalDump.options.source.connection.username = "${source_postgres_username}" | + .retrieval.spec.logicalDump.options.source.connection.password = "${source_postgres_password}" | + .retrieval.spec.logicalDump.options.parallelJobs = 1 + ' $dle_config_path/server.yml + + # restore pg_dump via pipe - without saving it on the disk + yq e -i ' + .databaseContainer.dockerImage="postgresai/extended-postgres:${source_postgres_version}" | + .retrieval.spec.logicalDump.options.immediateRestore.enabled=true | + .retrieval.spec.logicalDump.options.immediateRestore.forceInit=true | + .retrieval.spec.logicalDump.options.immediateRestore.configs alias = .databaseConfig | + del(.retrieval.jobs[] | select(. == "logicalRestore")) | + .databaseConfigs.configs.shared_preload_libraries = "${postgres_config_shared_preload_libraries}" + ' $dle_config_path/server.yml ;; s3) # Mount S3 bucket if it's defined in Terraform variables mkdir -p "${source_pgdump_s3_mount_point}" - s3fs ${source_pgdump_s3_bucket} ${source_pgdump_s3_mount_point} -o iam_role -o use_cache=/tmp -o allow_other + s3fs ${source_pgdump_s3_bucket} ${source_pgdump_s3_mount_point} -o iam_role -o allow_other - sed -ri "s/^(\s*)(- logicalDump.*$)/\1#- logicalDump /" ~/.dblab/server.yml - sed -ri "s|^(\s*)( dumpLocation:.*$)|\1 dumpLocation: ${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}|" ~/.dblab/server.yml + extra_mount="--volume ${source_pgdump_s3_mount_point}:${source_pgdump_s3_mount_point}" + + yq e -i ' + del(.retrieval.jobs[] | select(. == "logicalDump")) | + .retrieval.spec.logicalRestore.options.dumpLocation="${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}" + ' $dle_config_path/server.yml + + nProcessors=$(getconf _NPROCESSORS_ONLN) + yq e -i ' + .retrieval.spec.logicalDump.options.parallelJobs=${postgres_dump_parallel_jobs} | + .retrieval.spec.logicalRestore.options.parallelJobs=$nProcessors + ' $dle_config_path/server.yml ;; esac +# Fix ownership of the dblab directory +chown -R ubuntu.ubuntu /home/ubuntu/.dblab/ + sudo docker run \ - --name dblab_server \ - --label dblab_control \ - --privileged \ - --publish 2345:2345 \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - --volume /var/lib/dblab/dblab_pool_00/dump:/var/lib/dblab/dblab_pool/dump \ - --volume /var/lib/dblab:/var/lib/dblab/:rshared \ - --volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \ - --env DOCKER_API_VERSION=1.39 \ - --detach \ - --restart on-failure \ - postgresai/dblab-server:${dle_version_full} + --name dblab_server \ + --label dblab_control \ + --privileged \ + --publish 2345:2345 \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume /var/lib/dblab:/var/lib/dblab/:rshared \ + --volume $dle_config_path:/home/dblab/configs:ro \ + --volume $dle_meta_path:/home/dblab/meta \ + --volume $postgres_conf_path:/home/dblab/standard/postgres/control \ + $extra_mount \ + --env DOCKER_API_VERSION=1.39 \ + --detach \ + --restart on-failure \ +registry.gitlab.com/postgres-ai/database-lab/dblab-server:${dle_version} ### Waiting for the Database Lab Engine initialization. for i in {1..30000}; do @@ -162,42 +215,71 @@ for i in {1..30000}; do sleep 10 done -dblab init \ - --environment-id=tutorial \ - --url=http://localhost:2345 \ - --token=${dle_verification_token} \ - --insecure - -#configure and run Joe Bot container -cp /home/ubuntu/joe.yml ~/.dblab/joe.yml -sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)( token:.*$)/\1 token: ${platform_access_token}/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)( token:.*$)/\1 token: ${dle_verification_token}/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)( url:.*$)/\1 url: \"http\\:\\/\\/localhost\\:2345\"/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)(signingSecret:.*$)/\1signingSecret: ${platform_joe_signing_secret}/" ~/.dblab/joe.yml -sed -ri "s/^(\s*)(project:.*$)/\1project: ${platform_project_name}/" ~/.dblab/joe.yml +curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/scripts/cli_install.sh | bash +sudo mv ~/.dblab/dblab /usr/local/bin/dblab + +# Init dblab environment +su - ubuntu -c \ + 'dblab init \ + --environment-id=tutorial \ + --url=http://localhost:2345 \ + --token=${dle_verification_token} \ + --insecure' + +# Configure and run Joe Bot container. +joe_config_path="/home/ubuntu/.dblab/joe/configs" +joe_meta_path="/home/ubuntu/.dblab/joe/meta" + +mkdir -p $joe_config_path +mkdir -p $joe_meta_path + +curl https://gitlab.com/postgres-ai/joe/-/raw/${joe_version}/configs/config.example.yml --output $joe_config_path/joe.yml + +yq e -i ' + .app.debug = ${dle_debug_mode} | + .platform.token = "${platform_access_token}" | + .channelMapping.dblabServers.prod1.token = "${dle_verification_token}" | + .channelMapping.dblabServers.prod1.url = "http://localhost:2345/" | + .channelMapping.communicationTypes.webui[0].credentials.signingSecret = "${platform_joe_signing_secret}" | + .channelMapping.communicationTypes.webui[0].channels[0].project = "${platform_project_name}" | + .channelMapping.communicationTypes.webui[0].channels[0].dblabParams.dbname = "${source_postgres_dbname}" | + del(.channelMapping.communicationTypes.slack) | + del(.channelMapping.communicationTypes.slackrtm) | + del(.channelMapping.communicationTypes.slacksm) +' $joe_config_path/joe.yml sudo docker run \ - --name joe_bot \ - --network=host \ - --restart=on-failure \ - --volume ~/.dblab/joe.yml:/home/config/config.yml \ - --detach \ -postgresai/joe:latest - -#configure and run DB Migration Checker -curl https://gitlab.com/postgres-ai/database-lab/-/raw/master/configs/config.example.run_ci.yaml --output ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)( verificationToken: \"secret_token\".*$)/\1 verificationToken: ${vcs_db_migration_checker_verification_token}/" ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)( url: \"https\\:\\/\\/dblab.domain.com\"$)/\1 url: \"http\\:\\/\\/dblab_server\\:2345\"/" ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)( verificationToken: \"checker_secret_token\".*$)/\1 verificationToken: ${dle_verification_token}/" ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)( accessToken:.*$)/\1 accessToken: ${platform_access_token}/" ~/.dblab/run_ci.yaml -sed -ri "s/^(\s*)( token:.*$)/\1 token: ${vcs_github_secret_token}/" ~/.dblab/run_ci.yaml - -sudo docker run --name dblab_ci_checker -it --detach \ ---publish 2500:2500 \ ---volume /var/run/docker.sock:/var/run/docker.sock \ ---volume /tmp/ci_checker:/tmp/ci_checker \ ---volume ~/.dblab/run_ci.yaml:/home/dblab/configs/run_ci.yaml \ -postgresai/dblab-ci-checker:2.4.1 + --name joe_bot \ + --network=host \ + --restart=on-failure \ + --volume $joe_config_path:/home/configs:ro \ + --volume $joe_meta_path:/home/meta \ + --detach \ + registry.gitlab.com/postgres-ai/joe:${joe_version} + +# Configure and run DB Migration Checker. +ci_checker_config_path="/home/ubuntu/.dblab/ci_checker/configs" +mkdir -p $ci_checker_config_path + +curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/config.example.ci_checker.yml --output $ci_checker_config_path/ci_checker.yml + +yq e -i ' + .app.debug = ${dle_debug_mode} | + .app.verificationToken = "${vcs_db_migration_checker_verification_token}" | + .dle.url = "http://dblab_server:2345/" | + .dle.verificationToken = "${dle_verification_token}" | + .platform.accessToken = "${platform_access_token}" | + .source.token = "${vcs_github_secret_token}" +' $ci_checker_config_path/ci_checker.yml + +sudo docker run \ + --name dblab_ci_checker \ + --label dblab_control \ + --detach \ + --restart on-failure \ + --publish 2500:2500 \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume /tmp/ci_checker:/tmp/ci_checker \ + --volume $ci_checker_config_path:/home/dblab/configs:ro \ +registry.gitlab.com/postgres-ai/database-lab/dblab-ci-checker:${dle_version} + diff --git a/outputs.tf b/outputs.tf deleted file mode 100644 index 73c7edb..0000000 --- a/outputs.tf +++ /dev/null @@ -1,27 +0,0 @@ -output "aws_ec2_instance_ip" { - value = "${aws_instance.aws_ec2.public_ip}" -} -output "aws_ec2_instance_id" { - value = "${aws_instance.aws_ec2.id}" -} -output "aws_ec2_instance_dns" { - value = "${aws_instance.aws_ec2.public_dns}" -} -output "platform_dle_registration_url" { - value = "${format("%s://%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn))}" -} -output "platform_joe_registration_url" { - value = "${format("%s://%s:%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn),"444")}" -} -output "dle_verification_token" { - value = "${random_string.dle_verification_token.result}" -} -output "platform_joe_signing_secret" { - value = "${random_string.platform_joe_signing_secret.result}" -} -output "vcs_db_migration_checker_verification_token" { - value = "${random_string.vcs_db_migration_checker_verification_token.result}" -} -output "vcs_db_migration_checker_registration_url" { - value = "${format("%s://%s:%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn),"445")}" -} diff --git a/postgresql_clones_custom.conf b/postgresql_clones_custom.conf new file mode 100644 index 0000000..da1b437 --- /dev/null +++ b/postgresql_clones_custom.conf @@ -0,0 +1 @@ +log_min_duration_statement = 1000 diff --git a/terraform.tfstate b/terraform.tfstate deleted file mode 100644 index 7c49088..0000000 --- a/terraform.tfstate +++ /dev/null @@ -1,8 +0,0 @@ -{ - "version": 4, - "terraform_version": "0.15.3", - "serial": 49, - "lineage": "60a04a52-2c24-ec3b-9ab1-dd2c08731279", - "outputs": {}, - "resources": [] -} diff --git a/terraform.tfvars b/terraform.tfvars deleted file mode 100644 index f7aa1da..0000000 --- a/terraform.tfvars +++ /dev/null @@ -1,25 +0,0 @@ -dle_version_full = "2.4.1" - -aws_ami_name = "DBLABserver*" -aws_keypair = "YOUR_AWS_KEYPAIR" - -aws_deploy_region = "us-east-1" -aws_deploy_ebs_availability_zone = "us-east-1a" -aws_deploy_ec2_instance_type = "t2.large" -aws_deploy_ec2_instance_tag_name = "DBLABserver-ec2instance" -aws_deploy_ebs_size = "40" -aws_deploy_ebs_type = "gp2" -aws_deploy_allow_ssh_from_cidrs = ["0.0.0.0/0"] -aws_deploy_dns_api_subdomain = "tf-test" # subdomain in aws.postgres.ai, fqdn will be ${dns_api_subdomain}.aws.postgres.ai - -source_postgres_version = "13" -source_postgres_host = "ec2-3-215-57-87.compute-1.amazonaws.com" -source_postgres_port = "5432" -source_postgres_dbname = "d3dljqkrnopdvg" # this is an existing DB (Heroku example DB) -source_postgres_username = "postgres" - -dle_debug_mode = "true" -dle_retrieval_refresh_timetable = "0 0 * * 0" -postgres_config_shared_preload_libraries = "pg_stat_statements,logerrors" # DB Migration Checker requires logerrors extension - -platform_project_name = "aws_test_tf" diff --git a/ami.tf b/terraform/aws/ami.tf similarity index 100% rename from ami.tf rename to terraform/aws/ami.tf diff --git a/dns.tf b/terraform/aws/dns.tf similarity index 100% rename from dns.tf rename to terraform/aws/dns.tf diff --git a/instance.tf b/terraform/aws/instance.tf similarity index 57% rename from instance.tf rename to terraform/aws/instance.tf index 036b2bc..1dbd63b 100644 --- a/instance.tf +++ b/terraform/aws/instance.tf @@ -1,3 +1,14 @@ +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "aws_key_pair" "provision_key" { + key_name = "${var.aws_deploy_ec2_instance_tag_name}" + public_key = tls_private_key.ssh_key.public_key_openssh +} + + resource "random_string" "dle_verification_token" { length = 32 upper = true @@ -22,14 +33,20 @@ resource "random_string" "vcs_db_migration_checker_verification_token" { special = false } -data "template_file" "init" { - template = "${file("dle-logical-init.sh.tpl")}" - vars = { +resource "aws_instance" "aws_ec2" { + ami = "${data.aws_ami.ami.id}" + availability_zone = "${var.aws_deploy_ebs_availability_zone}" + instance_type = "${var.aws_deploy_ec2_instance_type}" + security_groups = ["${aws_security_group.dle_instance_sg.name}"] + key_name = aws_key_pair.provision_key.key_name + tags = "${local.common_tags}" + iam_instance_profile = "${var.source_type == "s3" ? "${aws_iam_instance_profile.instance_profile[0].name}" : null}" + user_data = templatefile("${path.module}/dle-logical-init.sh.tpl",{ dle_verification_token = "${random_string.dle_verification_token.result}" dle_debug_mode = "${var.dle_debug_mode}" dle_retrieval_refresh_timetable = "${var.dle_retrieval_refresh_timetable}" - dle_disks = "${join(" ",var.aws_deploy_ec2_volumes_names)}" - dle_version_full = "${var.dle_version_full}" + dle_version = "${var.dle_version}" + joe_version = "${var.joe_version}" aws_deploy_dns_zone_name = "${var.aws_deploy_dns_zone_name}" aws_deploy_dns_api_subdomain = "${var.aws_deploy_dns_api_subdomain}" aws_deploy_certificate_email = "${var.aws_deploy_certificate_email}" @@ -40,6 +57,7 @@ data "template_file" "init" { source_postgres_password = "${var.source_postgres_password}" source_postgres_version = "${var.source_postgres_version}" postgres_config_shared_preload_libraries = "${var.postgres_config_shared_preload_libraries}" + postgres_dump_parallel_jobs = "${var.postgres_dump_parallel_jobs}" platform_access_token = "${var.platform_access_token}" platform_project_name = "${var.platform_project_name}" platform_joe_signing_secret = "${random_string.platform_joe_signing_secret.result}" @@ -49,16 +67,41 @@ data "template_file" "init" { source_pgdump_s3_bucket = "${var.source_pgdump_s3_bucket}" source_pgdump_s3_mount_point = "${var.source_pgdump_s3_mount_point}" source_pgdump_path_on_s3_bucket = "${var.source_pgdump_path_on_s3_bucket}" + }) + + provisioner "local-exec" { # save private key locally + command = "echo '${tls_private_key.ssh_key.private_key_pem}' > ./${var.aws_deploy_ec2_instance_tag_name}.pem" + } + provisioner "local-exec" { + command = "chmod 600 ./'${var.aws_deploy_ec2_instance_tag_name}'.pem" } -} -resource "aws_instance" "aws_ec2" { - ami = "${data.aws_ami.ami.id}" - availability_zone = "${var.aws_deploy_ebs_availability_zone}" - instance_type = "${var.aws_deploy_ec2_instance_type}" - security_groups = ["${aws_security_group.dle_instance_sg.name}"] - key_name = "${var.aws_keypair}" - tags = "${local.common_tags}" - iam_instance_profile = "${var.source_type == "s3" ? "${aws_iam_instance_profile.instance_profile[0].name}" : null}" - user_data = "${data.template_file.init.rendered}" + provisioner "remote-exec" { + connection { + type = "ssh" + user = "ubuntu" + private_key = "${tls_private_key.ssh_key.private_key_pem}" + host = "${self.public_dns}" + } + inline = [ + "echo 'ssh is ready, will copy ssh public keys'", + "echo '${join("\n", var.ssh_public_keys_list)}' >> ~/.ssh/authorized_keys" + ] + } + + provisioner "local-exec" { + command = "for ssh_key in ${join(" ", var.ssh_public_keys_files_list)}; do cat $ssh_key | ssh -i ./${var.aws_deploy_ec2_instance_tag_name}.pem ubuntu@${self.public_dns} -o StrictHostKeyChecking=no 'cat >> ~/.ssh/authorized_keys'; done" + } + + provisioner "file" { + source = "postgresql_clones_custom.conf" + destination = "/tmp/postgresql_clones_custom.conf" + + connection { + type = "ssh" + user = "ubuntu" + private_key = "${tls_private_key.ssh_key.private_key_pem}" + host = "${self.public_dns}" + } + } } diff --git a/main.tf b/terraform/aws/main.tf similarity index 100% rename from main.tf rename to terraform/aws/main.tf diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf new file mode 100644 index 0000000..559497b --- /dev/null +++ b/terraform/aws/outputs.tf @@ -0,0 +1,58 @@ +output "aws_ec2_instance_ip" { + value = "${aws_instance.aws_ec2.public_ip}" +} +output "aws_ec2_instance_id" { + value = "${aws_instance.aws_ec2.id}" +} +output "aws_ec2_instance_dns" { + value = "${aws_instance.aws_ec2.public_dns}" +} +output "platform_dle_registration_url" { + value = "${format("%s://%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn))}" +} +output "platform_joe_registration_url" { + value = "${format("%s://%s:%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn),"444")}" +} +output "dle_verification_token" { + value = "${random_string.dle_verification_token.result}" +} +output "platform_joe_signing_secret" { + value = "${random_string.platform_joe_signing_secret.result}" +} +output "vcs_db_migration_checker_verification_token" { + value = "${random_string.vcs_db_migration_checker_verification_token.result}" +} +output "vcs_db_migration_checker_registration_url" { + value = "${format("%s://%s:%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn),"445")}" +} +output "embedded_ui_url" { + value = "${format("%s://%s:%s", "https",join("", aws_route53_record.dblab_subdomain.*.fqdn),"446")}" +} + +locals { + welcome_message = <