From dfa326eaf17a5631b67bfd90be529325afc12b8e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Fri, 2 Aug 2024 17:26:15 +0200 Subject: [PATCH 1/5] wip --- apps/certbot-service/.env.example | 9 ++ apps/certbot-service/Dockerfile | 47 ++++++++++ apps/certbot-service/README.md | 105 ++++++++++++++++++++++ apps/certbot-service/certbot.sh | 42 +++++++++ apps/certbot-service/deploy-hook.sh | 13 +++ apps/certbot-service/docker-compose.yml | 38 ++++++++ apps/certbot-service/entrypoint.sh | 42 +++++++++ apps/certbot-service/fly.toml | 2 + apps/db-service/.env.example | 2 +- apps/db-service/README.md | 6 ++ apps/db-service/entrypoint.sh | 4 +- apps/db-service/fly.toml | 2 +- apps/db-service/scripts/generate-certs.sh | 6 ++ apps/db-service/src/index.ts | 1 - 14 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 apps/certbot-service/.env.example create mode 100644 apps/certbot-service/Dockerfile create mode 100644 apps/certbot-service/README.md create mode 100644 apps/certbot-service/certbot.sh create mode 100644 apps/certbot-service/deploy-hook.sh create mode 100644 apps/certbot-service/docker-compose.yml create mode 100755 apps/certbot-service/entrypoint.sh create mode 100644 apps/certbot-service/fly.toml diff --git a/apps/certbot-service/.env.example b/apps/certbot-service/.env.example new file mode 100644 index 00000000..d3e932b0 --- /dev/null +++ b/apps/certbot-service/.env.example @@ -0,0 +1,9 @@ +CERTBOT_DOMAIN=db.postgres.new +CERTBOT_EMAIL="" +CLOUDFLARE_API_TOKEN="" +AWS_ACCESS_KEY_ID=minioadmin +AWS_ENDPOINT_URL_S3=http://minio:9000 +AWS_REGION=us-east-1 +AWS_SECRET_ACCESS_KEY=minioadmin +BUCKET_NAME=test +S3FS_MOUNT=/mnt/s3 \ No newline at end of file diff --git a/apps/certbot-service/Dockerfile b/apps/certbot-service/Dockerfile new file mode 100644 index 00000000..802aeb3e --- /dev/null +++ b/apps/certbot-service/Dockerfile @@ -0,0 +1,47 @@ +# syntax = docker/dockerfile:1 + +# Adjust CERTBOT_VERSION as desired +ARG CERTBOT_VERSION=2.11.0 +FROM certbot/dns-cloudflare:v${CERTBOT_VERSION} as base + +WORKDIR /app + +# Build S3FS +FROM base as build-s3fs + +# Install dependencies +RUN apk add --no-cache \ + git \ + build-base \ + automake \ + autoconf \ + libxml2-dev \ + fuse-dev \ + curl-dev + +RUN git clone https://github.com/s3fs-fuse/s3fs-fuse.git --branch v1.94 && \ + cd s3fs-fuse && \ + ./autogen.sh && \ + ./configure && \ + make && \ + make install + +# Final stage +FROM base + +# Install dependencies +RUN apk add --no-cache \ + bash \ + curl \ + fuse \ + libxml2 + +COPY --from=build-s3fs /usr/local/bin/s3fs /usr/local/bin/s3fs +COPY certbot.sh deploy-hook.sh entrypoint.sh /app/ + +RUN chmod +x certbot.sh +RUN chmod +x deploy-hook.sh + +ENTRYPOINT [ "./entrypoint.sh" ] + +CMD [ "./certbot.sh" ] diff --git a/apps/certbot-service/README.md b/apps/certbot-service/README.md new file mode 100644 index 00000000..d95254e5 --- /dev/null +++ b/apps/certbot-service/README.md @@ -0,0 +1,105 @@ +# Certbot + +This service is responsible for managing the certificates for the PGLite instances. + +It uses `fly machine run --schedule weekly` to wake up the service every week to renew the certificates if needed. Let's Encrypt certificates are valid for 90 days. + +## Testing certbot-service locally + +Copy `.env.example` to `.env` and set the missing environment variables. + +Start minio to emulate the S3 service: + +```shell +docker compose up -d minio +``` + +Initialize the bucket: + +```shell +docker compose up minio-init +``` + +Build and run the certbot service: + +```shell +docker compose up --build certbot-service +``` + +The certificates will be generated in `/mnt/s3/tls`. + +## Deploying to fly.io + +1. Create a new app if it doesn't exist + +```shell +flyctl apps create postgres-new-certbot +``` + +2. Build and deploy the Docker image to fly.io image registry + +```shell +flyctl deploy --build-only --push -a postgres-new-certbot --image-label + latest +``` + +3. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI (available in Bitwarden as a secure note "fly.io postgres.new cerbot .env") + +4. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`: + +```json +[ + { + "name": "postgres-new-certbot", + "app_name": "postgres-new-certbot", + "schedule": "0 0 1,15 * *", + "region": "ord", + "command": "./certbot.sh", + "command_timeout": 120, + "enabled": true, + "config": { + "metadata": { + "fly_process_group": "cron" + }, + "auto_destroy": true, + "disable_machine_autostart": true, + "guest": { + "cpu_kind": "shared", + "cpus": 1, + "memory_mb": 256 + }, + "image": "registry.fly.io/postgres-new-certbot:latest", + "restart": { + "max_retries": 1, + "policy": "no" + } + } + } +] +``` + +5. Test running the job by SSHing into cron-manager console + +Run this command in the cron-manager root folder: + +```shell +flyctl ssh console +``` + +Once in the cron-manager instance: + +```shell +cm jobs trigger 1 +``` + +If you open the "postgres-new-certbot" live logs in fly.io UI, you should see the job being executed. + +6. You can check if the certificates are present in the Tigris bucket + +Run this command in the apps/db-instance folder: + +```shell +flyctl storage dashboard +``` + +It should open the Tigris dashboard where you can check the bucket's content. The certificates should be created under `/tls`. \ No newline at end of file diff --git a/apps/certbot-service/certbot.sh b/apps/certbot-service/certbot.sh new file mode 100644 index 00000000..7743ae45 --- /dev/null +++ b/apps/certbot-service/certbot.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CONFIG_DIR="${S3FS_MOUNT}/tls/letsencrypt" +CERT_PATH="${CONFIG_DIR}/live/${CERTBOT_DOMAIN}/fullchain.pem" +CLOUD_FLARE_INI="/app/cloudflare.ini" +DEPLOY_HOOK="/app/deploy-hook.sh" + +renew_certificate() { + echo "Certificates exist. Renewing..." + certbot renew --non-interactive \ + --cert-name "${CERTBOT_DOMAIN}" \ + --dns-cloudflare \ + --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \ + --deploy-hook "${DEPLOY_HOOK}" \ + --config-dir "${CONFIG_DIR}" +} + +create_certificate() { + echo "Certificates do not exist. Creating..." + certbot certonly --non-interactive \ + --agree-tos \ + --cert-name "${CERTBOT_DOMAIN}" \ + --email "${CERTBOT_EMAIL}" \ + --dns-cloudflare \ + --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \ + --dns-cloudflare-propagation-seconds 60 \ + -d "*.${CERTBOT_DOMAIN}" \ + --deploy-hook "${DEPLOY_HOOK}" \ + --config-dir "${CONFIG_DIR}" +} + +main() { + if [[ -f "${CERT_PATH}" ]]; then + renew_certificate + else + create_certificate + fi +} + +main "$@" \ No newline at end of file diff --git a/apps/certbot-service/deploy-hook.sh b/apps/certbot-service/deploy-hook.sh new file mode 100644 index 00000000..1e135f3e --- /dev/null +++ b/apps/certbot-service/deploy-hook.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SOURCE_DIR="$S3FS_MOUNT/tls/letsencrypt/live/$CERTBOT_DOMAIN" +TARGET_DIR="$S3FS_MOUNT/tls" + +# Ensure the target directory exists +mkdir -p $TARGET_DIR + +# Copy the key and cert to the target directory +cp -f $SOURCE_DIR/privkey.pem $TARGET_DIR/key.pem +cp -f $SOURCE_DIR/fullchain.pem $TARGET_DIR/cert.pem \ No newline at end of file diff --git a/apps/certbot-service/docker-compose.yml b/apps/certbot-service/docker-compose.yml new file mode 100644 index 00000000..97d09f80 --- /dev/null +++ b/apps/certbot-service/docker-compose.yml @@ -0,0 +1,38 @@ +services: + certbot-service: + image: certbot-service + build: + context: . + env_file: + - .env + devices: + - /dev/fuse + cap_add: + - SYS_ADMIN + depends_on: + minio: + condition: service_healthy + minio: + image: minio/minio + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - 9000:9000 + - 9001:9001 + command: server /data --console-address :9001 + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 + interval: 5s + timeout: 5s + retries: 1 + minio-init: + image: minio/mc + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 minioadmin minioadmin; + (mc ls local/test || mc mb local/test); + " + depends_on: + minio: + condition: service_healthy diff --git a/apps/certbot-service/entrypoint.sh b/apps/certbot-service/entrypoint.sh new file mode 100755 index 00000000..fc083a79 --- /dev/null +++ b/apps/certbot-service/entrypoint.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# s3fs ################################ +cleanup() { + echo "Unmounting s3fs..." + fusermount -u $S3FS_MOUNT + exit 0 +} + +forward_signal() { + kill -$1 "$MAIN_PID" +} + +trap 'forward_signal SIGINT' SIGINT +trap 'forward_signal SIGTERM' SIGTERM +trap 'cleanup' EXIT + +# Create the mount point directory +mkdir -p $S3FS_MOUNT + +# Mount the S3 bucket +s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION + +# Check if the mount was successful +if mountpoint -q $S3FS_MOUNT; then + echo "S3 bucket mounted successfully at $S3FS_MOUNT" +else + echo "Failed to mount S3 bucket" + exit 1 +fi + +# cloudflare.ini ###################### +echo "dns_cloudflare_api_token = $CLOUDFLARE_API_TOKEN" > /app/cloudflare.ini +chmod 600 /app/cloudflare.ini + +# Execute the original command +"$@" & +MAIN_PID=$! + +wait $MAIN_PID diff --git a/apps/certbot-service/fly.toml b/apps/certbot-service/fly.toml new file mode 100644 index 00000000..483d258f --- /dev/null +++ b/apps/certbot-service/fly.toml @@ -0,0 +1,2 @@ +app = 'postgres-new-certbot' +primary_region = 'yyz' diff --git a/apps/db-service/.env.example b/apps/db-service/.env.example index 92e8b2ee..9efda7a4 100644 --- a/apps/db-service/.env.example +++ b/apps/db-service/.env.example @@ -6,5 +6,5 @@ AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin # Only if you need to test against a bucket hosted in AWS S3 # AWS_SESSION_TOKEN= -AWS_ENDPOINT_URL=http://minio:9000 +AWS_ENDPOINT_URL_S3=http://minio:9000 DOMAIN=*.db.example.com \ No newline at end of file diff --git a/apps/db-service/README.md b/apps/db-service/README.md index e208395b..1d3afcb0 100644 --- a/apps/db-service/README.md +++ b/apps/db-service/README.md @@ -83,3 +83,9 @@ To stop all Docker containers, run: ```shell docker compose down ``` + +## Deployment + +The db-service is deployed on Fly.io. + +A Tigris bucket is used to store the DB tarballs and the TLS certificates. \ No newline at end of file diff --git a/apps/db-service/entrypoint.sh b/apps/db-service/entrypoint.sh index c59c9b63..44b21866 100755 --- a/apps/db-service/entrypoint.sh +++ b/apps/db-service/entrypoint.sh @@ -5,7 +5,7 @@ set -o pipefail cleanup() { echo "Unmounting s3fs..." - fusermount -u $S3FS_MOUNT + umount $S3FS_MOUNT exit 0 } @@ -21,7 +21,7 @@ trap 'cleanup' EXIT mkdir -p $S3FS_MOUNT # Mount the S3 bucket -s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL -o endpoint=$AWS_REGION +s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION # Check if the mount was successful if mountpoint -q $S3FS_MOUNT; then diff --git a/apps/db-service/fly.toml b/apps/db-service/fly.toml index abe6e135..ea62e9ae 100644 --- a/apps/db-service/fly.toml +++ b/apps/db-service/fly.toml @@ -22,5 +22,5 @@ cpu_kind = 'shared' cpus = 1 [mounts] -source = "postgres-new-data" +source = "postgres_new_data" destination = "/mnt/data" diff --git a/apps/db-service/scripts/generate-certs.sh b/apps/db-service/scripts/generate-certs.sh index d6c12f04..a29ebe96 100755 --- a/apps/db-service/scripts/generate-certs.sh +++ b/apps/db-service/scripts/generate-certs.sh @@ -17,3 +17,9 @@ openssl genpkey -algorithm RSA -out key.pem openssl req -new -key key.pem -out csr.pem -subj "/CN=$DOMAIN" openssl x509 -req -in csr.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -days 365 + +# create fullchain by concatenating the server certificate and the CA certificate +cat cert.pem ca-cert.pem > fullchain.pem + +# replace cert with the fullchain +mv fullchain.pem cert.pem \ No newline at end of file diff --git a/apps/db-service/src/index.ts b/apps/db-service/src/index.ts index cbcccfc8..3a6211e8 100644 --- a/apps/db-service/src/index.ts +++ b/apps/db-service/src/index.ts @@ -24,7 +24,6 @@ await mkdir(tlsDir, { recursive: true }) const tls: TlsOptions = { key: await readFile(`${tlsDir}/key.pem`), cert: await readFile(`${tlsDir}/cert.pem`), - ca: await readFile(`${tlsDir}/ca-cert.pem`), } function getIdFromServerName(serverName: string) { From 65f5ba2dadba65abf4d556e396e73fbd23d5484a Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Aug 2024 10:17:14 +0200 Subject: [PATCH 2/5] fix config --- apps/db-service/fly.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/db-service/fly.toml b/apps/db-service/fly.toml index ea62e9ae..35897bc2 100644 --- a/apps/db-service/fly.toml +++ b/apps/db-service/fly.toml @@ -1,17 +1,17 @@ app = 'postgres-new-dev' primary_region = 'yyz' -[[service]] +[[services]] internal_port = 5432 protocol = "tcp" auto_stop_machines = true auto_start_machines = true min_machines_running = 0 -[[service.ports]] +[[services.ports]] port = 5432 -[[service.concurrency]] +[services.concurrency] type = "connections" hard_limit = 25 soft_limit = 20 From fb009b349fba4fede008eb402ce36a58f9172937 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Aug 2024 10:50:40 +0200 Subject: [PATCH 3/5] address comment --- apps/certbot-service/README.md | 4 +- package-lock.json | 120 +++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/apps/certbot-service/README.md b/apps/certbot-service/README.md index d95254e5..ece5f011 100644 --- a/apps/certbot-service/README.md +++ b/apps/certbot-service/README.md @@ -43,9 +43,9 @@ flyctl deploy --build-only --push -a postgres-new-certbot --image-label latest ``` -3. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI (available in Bitwarden as a secure note "fly.io postgres.new cerbot .env") +1. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI. -4. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`: +2. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`: ```json [ diff --git a/package-lock.json b/package-lock.json index bb867a7d..6f5f83b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1797,6 +1797,126 @@ "node": ">= 10" } }, + "node_modules/@next/swc-darwin-x64": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz", + "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz", + "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz", + "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz", + "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz", + "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz", + "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz", + "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz", + "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "license": "MIT", From 73620aa08898b479b4ec9cc295377eb30cc32cfe Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Aug 2024 12:38:36 +0200 Subject: [PATCH 4/5] fix wildcard --- apps/db-service/.env.example | 2 +- apps/db-service/scripts/generate-certs.sh | 2 +- apps/db-service/src/index.ts | 26 +++-- package-lock.json | 120 ++++++++++++++++++++++ 4 files changed, 141 insertions(+), 9 deletions(-) diff --git a/apps/db-service/.env.example b/apps/db-service/.env.example index 9efda7a4..c5cc9c83 100644 --- a/apps/db-service/.env.example +++ b/apps/db-service/.env.example @@ -7,4 +7,4 @@ AWS_SECRET_ACCESS_KEY=minioadmin # Only if you need to test against a bucket hosted in AWS S3 # AWS_SESSION_TOKEN= AWS_ENDPOINT_URL_S3=http://minio:9000 -DOMAIN=*.db.example.com \ No newline at end of file +WILDCARD_DOMAIN=db.example.com \ No newline at end of file diff --git a/apps/db-service/scripts/generate-certs.sh b/apps/db-service/scripts/generate-certs.sh index a29ebe96..a1b3d73a 100755 --- a/apps/db-service/scripts/generate-certs.sh +++ b/apps/db-service/scripts/generate-certs.sh @@ -4,7 +4,7 @@ set -e set -o pipefail S3FS_MOUNT=${S3FS_MOUNT:=.} -DOMAIN="${DOMAIN:=*.db.example.com}" +DOMAIN="*.${WILDCARD_DOMAIN:=db.example.com}" CERT_DIR="$S3FS_MOUNT/tls" mkdir -p $CERT_DIR diff --git a/apps/db-service/src/index.ts b/apps/db-service/src/index.ts index 21baf928..9d9b6018 100644 --- a/apps/db-service/src/index.ts +++ b/apps/db-service/src/index.ts @@ -1,6 +1,6 @@ import { PGlite, PGliteInterface } from '@electric-sql/pglite' import { vector } from '@electric-sql/pglite/vector' -import { mkdir, readFile, access } from 'node:fs/promises' +import { mkdir, readFile, access, rm } from 'node:fs/promises' import net from 'node:net' import { createReadStream } from 'node:fs' import { pipeline } from 'node:stream/promises' @@ -111,12 +111,24 @@ const server = net.createServer((socket) => { // Create a directory for the database await mkdir(dbPath, { recursive: true }); - // Extract the .tar.gz file - await pipeline( - createReadStream(dumpPath), - createGunzip(), - extract({ cwd: dbPath, }) - ); + try { + // Extract the .tar.gz file + await pipeline( + createReadStream(dumpPath), + createGunzip(), + extract({ cwd: dbPath }) + ); + } catch (error) { + console.error(error); + await rm(dbPath, { recursive: true, force: true }); // Clean up the partially created directory + connection.sendError({ + severity: 'FATAL', + code: 'XX000', + message: `Error extracting database: ${(error as Error).message}`, + }); + connection.socket.end(); + return; + } } db = new PGlite(dbPath, { diff --git a/package-lock.json b/package-lock.json index 3f6d625f..fb7fce9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13431,6 +13431,126 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } + }, + "node_modules/@next/swc-darwin-x64": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz", + "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-gnu": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz", + "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-arm64-musl": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz", + "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-gnu": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz", + "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-linux-x64-musl": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz", + "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-arm64-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz", + "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz", + "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@next/swc-win32-x64-msvc": { + "version": "14.2.3", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz", + "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } } } } From b17699bdcecce68109a8f1f21c71568607da7820 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 5 Aug 2024 12:56:06 +0200 Subject: [PATCH 5/5] gzip the tarball --- .../app/api/databases/[id]/upload/route.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/apps/postgres-new/app/api/databases/[id]/upload/route.ts b/apps/postgres-new/app/api/databases/[id]/upload/route.ts index 2f01bf6c..390780a0 100644 --- a/apps/postgres-new/app/api/databases/[id]/upload/route.ts +++ b/apps/postgres-new/app/api/databases/[id]/upload/route.ts @@ -1,15 +1,12 @@ -import { CompleteMultipartUploadCommandOutput, S3Client } from '@aws-sdk/client-s3' +import { S3Client } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { NextRequest } from 'next/server' +import { createGzip } from 'zlib' +import { Readable } from 'stream' const wildcardDomain = process.env.WILDCARD_DOMAIN ?? 'db.example.com' - -// The credentials are read from the environment automatically const s3Client = new S3Client({ endpoint: process.env.S3_ENDPOINT, forcePathStyle: true }) -/** - * Accepts a *.tar.gz database dump and streams contents to an S3-compatible bucket - */ export async function POST(req: NextRequest, { params }: { params: { id: string } }) { if (!req.body) { return new Response( @@ -26,12 +23,15 @@ export async function POST(req: NextRequest, { params }: { params: { id: string const databaseId = params.id const key = `dbs/${databaseId}.tar.gz` + const gzip = createGzip() + const body = Readable.from(streamToAsyncIterable(req.body)) + const upload = new Upload({ client: s3Client, params: { Bucket: process.env.S3_BUCKET, Key: key, - Body: req.body, + Body: body.pipe(gzip), }, }) @@ -46,4 +46,17 @@ export async function POST(req: NextRequest, { params }: { params: { id: string }), { headers: { 'content-type': 'application/json' } } ) +} + +async function* streamToAsyncIterable(stream: ReadableStream) { + const reader = stream.getReader() + try { + while (true) { + const { done, value } = await reader.read() + if (done) return + yield value + } + } finally { + reader.releaseLock() + } } \ No newline at end of file