chore - fly deployment

apps/certbot-service/.env.example

@@ -0,0 +1,9 @@
+CERTBOT_DOMAIN=db.postgres.new
+CERTBOT_EMAIL="<your-email>"
+CLOUDFLARE_API_TOKEN="<cloudflare-api-token>"
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_ENDPOINT_URL_S3=http://minio:9000
+AWS_REGION=us-east-1
+AWS_SECRET_ACCESS_KEY=minioadmin
+BUCKET_NAME=test
+S3FS_MOUNT=/mnt/s3

apps/certbot-service/Dockerfile

@@ -0,0 +1,47 @@
+# syntax = docker/dockerfile:1
+
+# Adjust CERTBOT_VERSION as desired
+ARG CERTBOT_VERSION=2.11.0
+FROM certbot/dns-cloudflare:v${CERTBOT_VERSION} as base
+
+WORKDIR /app
+
+# Build S3FS
+FROM base as build-s3fs
+
+# Install dependencies
+RUN apk add --no-cache \
+  git \
+  build-base \
+  automake \
+  autoconf \
+  libxml2-dev \
+  fuse-dev \
+  curl-dev
+
+RUN git clone https://github.com/s3fs-fuse/s3fs-fuse.git --branch v1.94 && \
+  cd s3fs-fuse && \
+  ./autogen.sh && \
+  ./configure && \
+  make && \
+  make install
+
+# Final stage
+FROM base
+
+# Install dependencies
+RUN apk add --no-cache \
+  bash \
+  curl \
+  fuse \
+  libxml2
+
+COPY --from=build-s3fs /usr/local/bin/s3fs /usr/local/bin/s3fs
+COPY certbot.sh deploy-hook.sh entrypoint.sh /app/
+
+RUN chmod +x certbot.sh
+RUN chmod +x deploy-hook.sh
+
+ENTRYPOINT [ "./entrypoint.sh" ]
+
+CMD [ "./certbot.sh" ]

apps/certbot-service/README.md

@@ -0,0 +1,105 @@
+# Certbot
+
+This service is responsible for managing the certificates for the PGLite instances.
+
+It uses `fly machine run --schedule weekly` to wake up the service every week to renew the certificates if needed. Let's Encrypt certificates are valid for 90 days.
+
+## Testing certbot-service locally
+
+Copy `.env.example` to `.env` and set the missing environment variables.
+
+Start minio to emulate the S3 service:
+
+```shell
+docker compose up -d minio    
+```
+
+Initialize the bucket:
+
+```shell
+docker compose up minio-init
+```
+
+Build and run the certbot service:
+
+```shell
+docker compose up --build certbot-service
+```
+
+The certificates will be generated in `/mnt/s3/tls`.
+
+## Deploying to fly.io
+
+1. Create a new app if it doesn't exist
+
+```shell
+flyctl apps create postgres-new-certbot
+```
+
+2. Build and deploy the Docker image to fly.io image registry
+
+```shell
+flyctl deploy --build-only --push -a postgres-new-certbot --image-label
+ latest
+```
+
+1. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI.
+
+2. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`:
+
+```json
+[
+  {
+      "name": "postgres-new-certbot",
+      "app_name": "postgres-new-certbot",
+      "schedule": "0 0 1,15 * *",
+      "region": "ord",
+      "command": "./certbot.sh",
+      "command_timeout": 120,
+      "enabled": true,
+      "config": {
+          "metadata": {
+              "fly_process_group": "cron"
+          },
+          "auto_destroy": true,
+          "disable_machine_autostart": true,
+          "guest": {
+              "cpu_kind": "shared",
+              "cpus": 1,
+              "memory_mb": 256
+          },
+          "image": "registry.fly.io/postgres-new-certbot:latest",
+          "restart": {
+              "max_retries": 1,
+              "policy": "no"
+          }
+      }
+  }
+]
+```
+
+5. Test running the job by SSHing into cron-manager console
+
+Run this command in the cron-manager root folder:
+
+```shell
+flyctl ssh console
+```
+
+Once in the cron-manager instance:
+
+```shell
+cm jobs trigger 1
+```
+
+If you open the "postgres-new-certbot" live logs in fly.io UI, you should see the job being executed.
+
+6. You can check if the certificates are present in the Tigris bucket
+
+Run this command in the apps/db-instance folder:
+
+```shell
+flyctl storage dashboard
+```
+
+It should open the Tigris dashboard where you can check the bucket's content. The certificates should be created under `/tls`.

apps/certbot-service/certbot.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+CONFIG_DIR="${S3FS_MOUNT}/tls/letsencrypt"
+CERT_PATH="${CONFIG_DIR}/live/${CERTBOT_DOMAIN}/fullchain.pem"
+CLOUD_FLARE_INI="/app/cloudflare.ini"
+DEPLOY_HOOK="/app/deploy-hook.sh"
+
+renew_certificate() {
+    echo "Certificates exist. Renewing..."
+    certbot renew --non-interactive \
+        --cert-name "${CERTBOT_DOMAIN}" \
+        --dns-cloudflare \
+        --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
+        --deploy-hook "${DEPLOY_HOOK}" \
+        --config-dir "${CONFIG_DIR}"
+}
+
+create_certificate() {
+    echo "Certificates do not exist. Creating..."
+    certbot certonly --non-interactive \
+        --agree-tos \
+        --cert-name "${CERTBOT_DOMAIN}" \
+        --email "${CERTBOT_EMAIL}" \
+        --dns-cloudflare \
+        --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
+        --dns-cloudflare-propagation-seconds 60 \
+        -d "*.${CERTBOT_DOMAIN}" \
+        --deploy-hook "${DEPLOY_HOOK}" \
+        --config-dir "${CONFIG_DIR}"
+}
+
+main() {
+    if [[ -f "${CERT_PATH}" ]]; then
+        renew_certificate
+    else
+        create_certificate
+    fi
+}
+
+main "$@"

apps/certbot-service/deploy-hook.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SOURCE_DIR="$S3FS_MOUNT/tls/letsencrypt/live/$CERTBOT_DOMAIN"
+TARGET_DIR="$S3FS_MOUNT/tls"
+
+# Ensure the target directory exists
+mkdir -p $TARGET_DIR
+
+# Copy the key and cert to the target directory
+cp -f $SOURCE_DIR/privkey.pem $TARGET_DIR/key.pem
+cp -f $SOURCE_DIR/fullchain.pem $TARGET_DIR/cert.pem

apps/certbot-service/docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  certbot-service:
+    image: certbot-service
+    build:
+      context: .
+    env_file:
+      - .env
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    depends_on:
+      minio:
+        condition: service_healthy
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: server /data --console-address :9001
+    healthcheck:
+      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 1
+  minio-init:
+    image: minio/mc
+    entrypoint: >
+      /bin/sh -c "
+        mc alias set local http://minio:9000 minioadmin minioadmin;
+        (mc ls local/test || mc mb local/test);
+      "
+    depends_on:
+      minio:
+        condition: service_healthy

apps/certbot-service/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# s3fs ################################
+cleanup() {
+  echo "Unmounting s3fs..."
+  fusermount -u $S3FS_MOUNT
+  exit 0
+}
+
+forward_signal() {
+  kill -$1 "$MAIN_PID"
+}
+
+trap 'forward_signal SIGINT' SIGINT
+trap 'forward_signal SIGTERM' SIGTERM
+trap 'cleanup' EXIT
+
+# Create the mount point directory
+mkdir -p $S3FS_MOUNT
+
+# Mount the S3 bucket
+s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION
+
+# Check if the mount was successful
+if mountpoint -q $S3FS_MOUNT; then
+  echo "S3 bucket mounted successfully at $S3FS_MOUNT"
+else
+  echo "Failed to mount S3 bucket"
+  exit 1
+fi
+
+# cloudflare.ini ######################
+echo "dns_cloudflare_api_token = $CLOUDFLARE_API_TOKEN" > /app/cloudflare.ini
+chmod 600 /app/cloudflare.ini
+
+# Execute the original command
+"$@" &
+MAIN_PID=$!
+
+wait $MAIN_PID

apps/certbot-service/fly.toml

@@ -0,0 +1,2 @@
+app = 'postgres-new-certbot'
+primary_region = 'yyz'

apps/db-service/.env.example

@@ -6,5 +6,5 @@ AWS_ACCESS_KEY_ID=minioadmin
 AWS_SECRET_ACCESS_KEY=minioadmin
 # Only if you need to test against a bucket hosted in AWS S3
 # AWS_SESSION_TOKEN=<aws-session-token>
-AWS_ENDPOINT_URL=http://minio:9000
-DOMAIN=*.db.example.com
+AWS_ENDPOINT_URL_S3=http://minio:9000
+WILDCARD_DOMAIN=db.example.com

apps/db-service/README.md

@@ -83,3 +83,9 @@ To stop all Docker containers, run:
 ```shell
 docker compose down
 ```
+
+## Deployment
+
+The db-service is deployed on Fly.io.
+
+A Tigris bucket is used to store the DB tarballs and the TLS certificates.

apps/db-service/entrypoint.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 cleanup() {
   echo "Unmounting s3fs..."
-  fusermount -u $S3FS_MOUNT
+  umount $S3FS_MOUNT
   exit 0
 }
 
@@ -21,7 +21,7 @@ trap 'cleanup' EXIT
 mkdir -p $S3FS_MOUNT
 
 # Mount the S3 bucket
-s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL -o endpoint=$AWS_REGION
+s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION
 
 # Check if the mount was successful
 if mountpoint -q $S3FS_MOUNT; then

apps/db-service/fly.toml

@@ -1,17 +1,17 @@
 app = 'postgres-new-dev'
 primary_region = 'yyz'
 
-[[service]]
+[[services]]
 internal_port = 5432
 protocol = "tcp"
 auto_stop_machines = true
 auto_start_machines = true
 min_machines_running = 0
 
-[[service.ports]]
+[[services.ports]]
 port = 5432
 
-[[service.concurrency]]
+[services.concurrency]
 type = "connections"
 hard_limit = 25
 soft_limit = 20
@@ -22,5 +22,5 @@ cpu_kind = 'shared'
 cpus = 1
 
 [mounts]
-source = "postgres-new-data"
+source = "postgres_new_data"
 destination = "/mnt/data"

apps/db-service/scripts/generate-certs.sh

@@ -4,7 +4,7 @@ set -e
 set -o pipefail
 
 S3FS_MOUNT=${S3FS_MOUNT:=.}
-DOMAIN="${DOMAIN:=*.db.example.com}"
+DOMAIN="*.${WILDCARD_DOMAIN:=db.example.com}"
 CERT_DIR="$S3FS_MOUNT/tls"
 
 mkdir -p $CERT_DIR
@@ -17,3 +17,9 @@ openssl genpkey -algorithm RSA -out key.pem
 openssl req -new -key key.pem -out csr.pem -subj "/CN=$DOMAIN"
 
 openssl x509 -req -in csr.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -days 365
+
+# create fullchain by concatenating the server certificate and the CA certificate
+cat cert.pem ca-cert.pem > fullchain.pem
+
+# replace cert with the fullchain
+mv fullchain.pem cert.pem