From c70a74e009e8e4b74ee6e6d3c952f7e378857e13 Mon Sep 17 00:00:00 2001 From: Wesley Schwengle Date: Thu, 16 Sep 2021 21:02:38 -0400 Subject: [PATCH 01/45] Add postgres login details to docker-compose.yml Resolves: gh#70 Signed-off-by: Wesley Schwengle --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 1a6d245..59c5f3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -366,6 +366,8 @@ services: PG_TAG: "${PG_VERSION_TAG:-9.6-alpine}" environment: POSTGRES_PASSWORD: metacpan + POSTGRES_USERNAME: metacpan123 + POSTGRES_DB: metacpan networks: - database healthcheck: From ff4160f37dae9f35bdf1a09f12906ee9e5a78e29 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 30 Oct 2021 21:15:14 -0400 Subject: [PATCH 02/45] Fix quoting setting environment variables Docker started throwing error messages because of the dashes (`-`) that appear as part of of the environment variables being set. Adding double quotes around the setting fixes the error. --- .env | 2 +- grep.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 074398d..8ea602e 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ COMPOSE_PROJECT_NAME=metacpan PLACK_ENV=development PGDB=pgdb:5432 -API_SERVER=morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose +API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose" diff --git a/grep.env b/grep.env index 6b03c65..8e1d6a5 100644 --- a/grep.env +++ b/grep.env @@ -1,4 +1,4 @@ # grep service - development environment GREP_SITE_PORT=3001 -GREP_PLACKUP_SERVER_ARGS=-E development -R lib,bin \ No newline at end of file +GREP_PLACKUP_SERVER_ARGS="-E development -R lib,bin" From 0cf6a12ae161cc86306feb14759b7788e3a5c547 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Tue, 27 Dec 2022 13:35:48 +0300 Subject: [PATCH 03/45] set MOJO_MODE to 'testing' to mark the environment --- .circleci/config.yml | 3 +++ .env | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a7ca498..1760122 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,9 +36,12 @@ jobs: ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test -- name: wait for ES - run: + name: Run complete MetaCPAN API Test Suite command: | docker-compose exec -T api_test prove -lr --jobs 2 t docker-compose down + environment: + MOJO_MODE: testing - run: command: | docker-compose logs diff --git a/.env b/.env index 8ea602e..074398d 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ COMPOSE_PROJECT_NAME=metacpan PLACK_ENV=development PGDB=pgdb:5432 -API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose" +API_SERVER=morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose From 5984be8ef20c5a6cb0867f8542f1fafa25431987 Mon Sep 17 00:00:00 2001 From: Bodo Hugo Barwich Date: Fri, 30 Dec 2022 17:04:46 +0300 Subject: [PATCH 04/45] set MOJO_MODE for Docker environment --- .circleci/config.yml | 3 +-- localapi.env | 1 + localapi_test.env | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1760122..16b113f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,8 @@ jobs: command: | docker-compose exec -T api_test prove -lr --jobs 2 t docker-compose down - environment: - MOJO_MODE: testing - run: + name: Show Docker container logs on Error command: | docker-compose logs docker stats --no-stream diff --git a/localapi.env b/localapi.env index f0a4aa4..97514a8 100644 --- a/localapi.env +++ b/localapi.env @@ -6,3 +6,4 @@ ES_TEST=elasticsearch_test:9200 MINICPAN=/CPAN PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton +MOJO_MODE=development diff --git a/localapi_test.env b/localapi_test.env index 4ea8bc2..1f49954 100644 --- a/localapi_test.env +++ b/localapi_test.env @@ -8,3 +8,4 @@ METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing MINICPAN=/CPAN PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton +MOJO_MODE=testing From 32455840b729cb5676122f2c07b8273770e1a0f1 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 27 Apr 2023 14:46:45 +0200 Subject: [PATCH 05/45] Bump actions/checkout to v3 --- .github/workflows/up.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/up.yml b/.github/workflows/up.yml index 9d6a13f..e3b1a6b 100644 --- a/.github/workflows/up.yml +++ b/.github/workflows/up.yml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 1 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: init run: bin/metacpan-docker init From 49395532c736feb75b2573876dc95bd6501621f6 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 27 Apr 2023 14:50:01 +0200 Subject: [PATCH 06/45] Remove duplicate key in CircleCI config --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16b113f..90564db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,6 @@ jobs: docker-compose logs docker stats --no-stream docker ps -a | head - name: docker-compose logs when: on_fail - run: command: | From d2a23f3cafb103777eb52b17ceee42e7eb21baae Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 27 Apr 2023 16:46:19 +0200 Subject: [PATCH 07/45] Don't try to nest read-only mounts in Docker At some point Docker stopped permitting this. Disable the read-only attribute on metacpan-web in order to be able to nest mounts. The proper change would probably be to rearchitect where the local config file lives. --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6ddb536..3ec32ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,13 +71,13 @@ services: - type: volume source: web_carton target: /carton - - type: bind - source: ./configs/metacpan-web/metacpan_web_local.conf - target: /metacpan-web/metacpan_web_local.conf - read_only: true - type: bind source: ./src/metacpan-web target: /metacpan-web + # read_only: true + - type: bind + source: ./configs/metacpan-web/metacpan_web_local.conf + target: /metacpan-web/metacpan_web_local.conf read_only: true ports: - "5001:5001" From 4397dc028974b7bdc36f2f81662cfa206f8ec476 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 30 Jan 2024 16:51:56 -0500 Subject: [PATCH 08/45] Fix typos --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d102ee..6345f5c 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ You can access the dashboard configuration via: #### `web` -The local instance of the web front end is accessiable via: +The local instance of the web front end is accessible via: * [http://localhost:5001](http://localhost:5001) * [http://web.metacpan.localhost](http://web.metacpan.localhost) @@ -313,7 +313,7 @@ This will stay on your current local branch. This is used to reset all the git repositories in `src/*` to their latest version on `upstream/master`. -This will fail if you have some uncommited local changes. +This will fail if you have some uncommitted local changes. You should then commit or cancel your changes before re-running the command. #### `bin/metacpan-docker` build/up/down/start/stop/run/ps/top... From 04e477a4515ab1e5c9d947b6811c83ebb0b5111b Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 26 Apr 2024 14:00:32 +0100 Subject: [PATCH 09/45] add metacpan-ingest --- bin/metacpan-docker | 2 +- .../metacpan-ingest/metacpan_ingest_local.conf | 1 + docker-compose.yml | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 configs/metacpan-ingest/metacpan_ingest_local.conf diff --git a/bin/metacpan-docker b/bin/metacpan-docker index c3ea546..d8ebd7a 100755 --- a/bin/metacpan-docker +++ b/bin/metacpan-docker @@ -3,7 +3,7 @@ set -e -GitRepos=("metacpan-api" "metacpan-web" "metacpan-grep-front-end" "metacpan-cpan-extracted-lite") +GitRepos=("metacpan-api" "metacpan-web" "metacpan-grep-front-end" "metacpan-cpan-extracted-lite" "metacpan-ingest") # sanity check type "docker" > /dev/null diff --git a/configs/metacpan-ingest/metacpan_ingest_local.conf b/configs/metacpan-ingest/metacpan_ingest_local.conf new file mode 100644 index 0000000..aed640c --- /dev/null +++ b/configs/metacpan-ingest/metacpan_ingest_local.conf @@ -0,0 +1 @@ +cpan = /CPAN diff --git a/docker-compose.yml b/docker-compose.yml index 3ec32ac..fd32fe6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -198,6 +198,21 @@ services: - database - elasticsearch + ingest: + image: metacpan/metacpan-ingest:latest + volumes: + - type: volume + source: cpan + target: /CPAN + - type: bind + source: ./configs/metacpan-ingest/metacpan_ingest_local.conf + target: /metacpan-ingest/metacpan_ingest_local.conf + read_only: true + depends_on: + - elasticsearch + networks: + - elasticsearch + # _ _ _ _ _ # __ _(_) |_| |__ _ _| |__ _ __ ___ ___ ___| |_ ___ # / _` | | __| '_ \| | | | '_ \ | '_ ` _ \ / _ \/ _ \ __/ __| From cf53558b68cc993a5c33e16d7d2e09fbc945c09b Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 26 Apr 2024 17:00:26 +0100 Subject: [PATCH 10/45] update readme to cover buildx, buildkit, and ES 2.4 on mac systems --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 6345f5c..35a5249 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,19 @@ access docker follow the This document assumes the post-installation steps have been followed for the current user. +You will also need Docker buildx, and to enable Docker BuildKit. They should +be set up by default when using Docker Desktop, but on Linux you may need to +install them. buildx is the `docker-buildx` package on Debian based systems. +Docker BuildKit can be enabled by following the +[Getting Started](https://docs.docker.com/build/buildkit/#getting-started) +instructions. + +If you are running a Mac ARM64 system, you will need to manually tell docker +to use the x86_64 version of Elasticsearch 2.4. This can be done by running +the command: + + docker pull elasticsearch:2.4 --platform=linux/x86_64 + It is highly recommended that you alias `docker-compose` to `fig` (its original name) and use it wherever `docker-compose` is used. You are going to have to type this command a lot. From 516cfed6a99400be330b1c9d97688017b25f13a5 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2024 10:24:39 +0100 Subject: [PATCH 11/45] Remove github-meets-cpan --- .circleci/config.yml | 8 --- .github/workflows/up.yml | 10 ---- README.md | 71 +++++++++++--------------- docker-compose.yml | 107 --------------------------------------- 4 files changed, 29 insertions(+), 167 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90564db..0a92c0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,11 +47,3 @@ jobs: docker stats --no-stream docker ps -a | head when: on_fail - - run: - command: | - docker-compose --verbose up -d github-meets-cpan - name: github-meets-cpan up - - run: - command: | - docker-compose exec -T github-meets-cpan prove -lr --jobs 2 t - docker-compose down diff --git a/.github/workflows/up.yml b/.github/workflows/up.yml index e3b1a6b..0adfc79 100644 --- a/.github/workflows/up.yml +++ b/.github/workflows/up.yml @@ -28,16 +28,6 @@ jobs: - name: down run: docker-compose down - - name: up github-meets-cpan - run: docker-compose up -d github-meets-cpan - - name: down - run: docker-compose down - - - name: up github-meets-cpan-cron - run: docker-compose up -d github-meets-cpan-cron - - name: down - run: docker-compose down - - name: up web run: docker-compose up -d web - name: down diff --git a/README.md b/README.md index 35a5249..f0cfa63 100644 --- a/README.md +++ b/README.md @@ -9,35 +9,35 @@ * [Running the MetaCPAN stack with Docker (via Docker Compose)](#running-the-metacpan-stack-with-docker-via-docker-compose) * [Quick Start](#quick-start) * [Working with Containers](#working-with-containers) - * [Building Containers](#building-containers) - * [Accessing Containers](#accessing-containers) - * [Accessing Services](#accessing-services) - * [`web`](#web) - * [`api`](#api) - * [`github-meets-cpan`](#github-meets-cpan) - * [`Elasticsearch`](#elasticsearch) - * [`grep`](#grep) + * [Building Containers](#building-containers) + * [Accessing Containers](#accessing-containers) + * [Accessing Services](#accessing-services) + * [`web`](#web) + * [`api`](#api) + * [`Elasticsearch`](#elasticsearch) + * [`PostgreSQL`](#postgresql) + * [`grep`](#grep) * [System architecture](#system-architecture) - * [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script) - * [`bin/metacpan init`](#binmetacpan-init) - * [`bin/metacpan localapi`](#binmetacpan-localapi) - * [`bin/metacpan-docker pull`](#binmetacpan-docker-pull) - * [`bin/metacpan-docker reset`](#binmetacpan-docker-reset) - * [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop) - * [Services](#services) - * [`web`](#web-1) - * [`api`](#api-1) - * [`grep`](#grep-1) - * [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service) - * [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices) - * [Putting the above all together](#putting-the-above-all-together) - * [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test) + * [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script) + * [`bin/metacpan init`](#binmetacpan-init) + * [`bin/metacpan localapi`](#binmetacpan-localapi) + * [`bin/metacpan-docker pull`](#binmetacpan-docker-pull) + * [`bin/metacpan-docker reset`](#binmetacpan-docker-reset) + * [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop) + * [Services](#services) + * [`web`](#web-1) + * [`api`](#api-1) + * [`grep`](#grep-1) + * [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service) + * [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices) + * [Putting the above all together](#putting-the-above-all-together) + * [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test) * [Tips and tricks](#tips-and-tricks) - * [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker) - * [Running tests](#running-tests) - * [Updating Carton dependencies](#updating-carton-dependencies) - * [Updating the git repositories](#updating-the-git-repositories) - * [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data) + * [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker) + * [Running tests](#running-tests) + * [Updating Carton dependencies](#updating-carton-dependencies) + * [Updating the git repositories](#updating-the-git-repositories) + * [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data) * [Peeking Inside the Container](#peeking-inside-the-container) * [To Do](#to-do) * [See also](#see-also) @@ -173,11 +173,6 @@ To access the `psql` command line client in the PostgreSQL container: docker-compose exec pgdb psql -Similarly the `mongodb` command line client in the MongoDB container is accessed -via: - - docker-compose exec mongodb mongo --host mongodb test - ### Accessing Services Each container is responsible for a different service. Some of these services @@ -188,7 +183,6 @@ The current configuration is: - api: [http://api.metacpan.localhost](http://api.metacpan.localhost) - web: [http://web.metacpan.localhost](http://web.metacpan.localhost) -- github-meets-cpan: [http://gh.metacpan.localhost](http://gh.metacpan.localhost) - grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) In order to access to the localhost subdomains, you probably have to manually @@ -220,11 +214,6 @@ The local instance of the web front end is accessible via: * [http://localhost:5000](http://localhost:5000) * [http://api.metacpan.localhost](http://api.metacpan.localhost) -#### `github-meets-cpan` - -* [http://localhost:3000](http://localhost:3000) -* [http://gh.metacpan.localhost](http://gh.metacpan.localhost) - #### `Elasticsearch` The `elasticsearch` and `elasticsearch_test` containers are not exposed directly. They are available via the `api` and `api_test` containers. @@ -241,10 +230,9 @@ You can query the `elasticsearch_test` container via: docker-compose exec elasticsearch_test curl http://localhost:9200 ``` -#### `PostgreSQL and MongoDB` +#### `PostgreSQL` -The PostgreSQL and MongoDB services by default are only accessible from other -containers. +The PostgreSQL service by default is only accessible from other containers. #### `grep` @@ -266,7 +254,6 @@ The system consists of several services that live in docker containers: - `elasticsearch_test` — database for `api_test` - `pgdb` - PostgreSQL database container - `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) -- `github-meets-cpan` - Containerized version of [gh.metacpan.org](https://gh.metacpan.org) - `grep` - the web interface for grep.metacpan on [http://localhost:3001](http://localhost:3001) These services use one or more Docker volumes: diff --git a/docker-compose.yml b/docker-compose.yml index fd32fe6..6059aaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,93 +213,6 @@ services: networks: - elasticsearch - # _ _ _ _ _ - # __ _(_) |_| |__ _ _| |__ _ __ ___ ___ ___| |_ ___ - # / _` | | __| '_ \| | | | '_ \ | '_ ` _ \ / _ \/ _ \ __/ __| - # | (_| | | |_| | | | |_| | |_) | | | | | | | __/ __/ |_\__ \ - # \__, |_|\__|_| |_|\__,_|_.__/ |_| |_| |_|\___|\___|\__|___/ - # |___/ - # - # ___ _ __ __ _ _ __ - # / __| '_ \ / _` | '_ \ - # | (__| |_) | (_| | | | | - # \___| .__/ \__,_|_| |_| - # |_| - # - github-meets-cpan: - image: metacpan/github-meets-cpan:latest - command: "/wait-for-it.sh mongodb:27017 -- morbo script/app.pl" - depends_on: - - mongodb - - logspout - - traefik - networks: - - mongo - - traefik-network - labels: - - "traefik.enable=true" - - "traefik.docker.network=traefik-network" - - "traefik.http.routers.github-meets-cpan.rule=Host(`gh.metacpan.localhost`)" - - "traefik.http.services.gh-meet-cpan-web.loadbalancer.server.port=3000" - - # _ _ _ _ _ - # __ _(_) |_| |__ _ _| |__ _ __ ___ ___ ___| |_ ___ - # / _` | | __| '_ \| | | | '_ \ | '_ ` _ \ / _ \/ _ \ __/ __| - # | (_| | | |_| | | | |_| | |_) | | | | | | | __/ __/ |_\__ \ - # \__, |_|\__|_| |_|\__,_|_.__/ |_| |_| |_|\___|\___|\__|___/ - # |___/ - # - # ___ _ __ __ _ _ __ ___ _ __ ___ _ __ - # / __| '_ \ / _` | '_ \ / __| '__/ _ \| '_ \ - # | (__| |_) | (_| | | | | | (__| | | (_) | | | | - # \___| .__/ \__,_|_| |_| \___|_| \___/|_| |_| - # |_| - # - - github-meets-cpan-cron: - image: metacpan/github-meets-cpan:latest - command: "/wait-for-it.sh mongodb:27017 -- perl cron/update.pl" - depends_on: - - mongodb - volumes: - - type: bind - source: ${MC_CONF_PRIVATE_DIR:-.}/github-meets-cpan/environment.json - target: /code/environment.json - read_only: true - networks: - - mongo - - # __ _ _ __ ___ _ __ - # / _` | '__/ _ \ '_ \ - # | (_| | | | __/ |_) | - # \__, |_| \___| .__/ - # __/ | | | - # |___/ |_| - - grep: - depends_on: - - traefik - image: metacpan/metacpan-grep-front-end:latest - build: - context: ./src/metacpan-grep-front-end - volumes: - - type: volume - source: metacpan_git_shared - target: /shared/metacpan_git - read_only: true - - type: bind - source: ./src/metacpan-grep-front-end - target: /metacpan-grep-front-end - read_only: true - env_file: - - grep.env - networks: - - traefik-network - labels: - - "traefik.enable=true" - - "traefik.http.routers.grep.rule=Host(`grep.metacpan.localhost`)" - - "traefik.http.services.grep-web.loadbalancer.server.port=3000" - # ____ _ _____ _ ____ _ ____ _____ ____ # | _ \ / \|_ _|/ \ | __ ) / \ / ___|| ____/ ___| # | | | |/ _ \ | | / _ \ | _ \ / _ \ \___ \| _| \___ \ @@ -404,25 +317,6 @@ services: target: /healthcheck.sh read_only: true - # _ _ - # _ __ ___ ___ _ __ __ _ ___ __| | |__ - # | '_ ` _ \ / _ \| '_ \ / _` |/ _ \ / _` | '_ \ - # | | | | | | (_) | | | | (_| | (_) | (_| | |_) | - # |_| |_| |_|\___/|_| |_|\__, |\___/ \__,_|_.__/ - # |___/ - # - - mongodb: - image: mongo:4.4.9 - networks: - - mongo - healthcheck: - interval: 10s - timeout: 10s - retries: 0 - start_period: 40s - test: echo 'db.runCommand("ping").ok' | mongo mongodb:27017/test --quiet - # _ _ _____ _______ _____ ____ _ ______ # | \ | | ____|_ _\ \ / / _ \| _ \| |/ / ___| # | \| | _| | | \ \ /\ / / | | | |_) | ' /\___ \ @@ -433,7 +327,6 @@ services: networks: database: elasticsearch: - mongo: traefik-network: web-network: From 4b5d47f6b92da2eebc62eeeae09f5104852faa7c Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2024 11:12:58 +0100 Subject: [PATCH 12/45] Restore grep config after accidentally removing it --- docker-compose.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 6059aaa..3a47f53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,6 +213,37 @@ services: networks: - elasticsearch + # __ _ _ __ ___ _ __ + # / _` | '__/ _ \ '_ \ + # | (_| | | | __/ |_) | + # \__, |_| \___| .__/ + # __/ | | | + # |___/ |_| + + grep: + depends_on: + - traefik + image: metacpan/metacpan-grep-front-end:latest + build: + context: ./src/metacpan-grep-front-end + volumes: + - type: volume + source: metacpan_git_shared + target: /shared/metacpan_git + read_only: true + - type: bind + source: ./src/metacpan-grep-front-end + target: /metacpan-grep-front-end + read_only: true + env_file: + - grep.env + networks: + - traefik-network + labels: + - "traefik.enable=true" + - "traefik.http.routers.grep.rule=Host(`grep.metacpan.localhost`)" + - "traefik.http.services.grep-web.loadbalancer.server.port=3000" + # ____ _ _____ _ ____ _ ____ _____ ____ # | _ \ / \|_ _|/ \ | __ ) / \ / ___|| ____/ ___| # | | | |/ _ \ | | / _ \ | _ \ / _ \ \___ \| _| \___ \ From 49a7884872b497b1cddd9a72ab2e1974ee094078 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2024 15:44:35 +0100 Subject: [PATCH 13/45] Tidy some of README --- README.md | 75 +++++++++++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index f0cfa63..31cd866 100644 --- a/README.md +++ b/README.md @@ -92,10 +92,10 @@ Then, clone this repo and set up the environment: The `bin/metacpan-docker init` command clones the source repositories for: -- `metacpan-web` -- `metacpan-api` -- `metacpan-grep-front-end` -- `metacpan-cpan-extracted-lite` +* `metacpan-web` +* `metacpan-api` +* `metacpan-grep-front-end` +* `metacpan-cpan-extracted-lite` These repositories are automatically mounted into the appropriate docker containers allowing the developer to use their preferred tools to work with the @@ -118,8 +118,8 @@ following command in a separate terminal to get yourself up to speed: docker-compose exec api index-cpan.sh This will prompt you to confirm removing old indices and setting up mappings on -the Elasticsearch service (say `YES`). It will then proceed to rsync a partial CPAN in -`/CPAN` for its metadata to be imported. +the Elasticsearch service (say `YES`). It will then proceed to rsync a partial +CPAN in `/CPAN` for its metadata to be imported. Once the above is done, you should be able to see your local partial CPAN data in e.g. [http://localhost:5001/recent](http://localhost:5001/recent) and @@ -150,9 +150,7 @@ For further details, read on! You can (re)build arbitrary containers. For instance, if you want to rebuild the `api` container: -``` -docker-compose build api -``` + docker-compose build api ### Accessing Containers @@ -181,21 +179,19 @@ are available in the developer environment via ports on the host system. We are using [traefik][13] to manage the traffic between services. The current configuration is: -- api: [http://api.metacpan.localhost](http://api.metacpan.localhost) -- web: [http://web.metacpan.localhost](http://web.metacpan.localhost) -- grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) +* api: [http://api.metacpan.localhost](http://api.metacpan.localhost) +* web: [http://web.metacpan.localhost](http://web.metacpan.localhost) +* grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) In order to access to the localhost subdomains, you probably have to manually add these entries in you `/etc/hosts` file. -``` -# add to /etc/hosts -127.0.0.1 api.metacpan.localhost -127.0.0.1 gh.metacpan.localhost -127.0.0.1 grep.metacpan.localhost -127.0.0.1 metacpan.localhost -127.0.0.1 web.metacpan.localhost -``` + # add to /etc/hosts + 127.0.0.1 api.metacpan.localhost + 127.0.0.1 gh.metacpan.localhost + 127.0.0.1 grep.metacpan.localhost + 127.0.0.1 metacpan.localhost + 127.0.0.1 web.metacpan.localhost You can access the dashboard configuration via: [http://metacpan.localhost:8080](http://metacpan.localhost:8080) @@ -216,19 +212,16 @@ The local instance of the web front end is accessible via: #### `Elasticsearch` -The `elasticsearch` and `elasticsearch_test` containers are not exposed directly. They are available via the `api` and `api_test` containers. +The `elasticsearch` and `elasticsearch_test` containers are not exposed +directly. They are available via the `api` and `api_test` containers. You can query the `elasticsearch` container via: -``` -docker-compose exec elasticsearch curl http://localhost:9200 -``` + docker-compose exec elasticsearch curl http://localhost:9200 You can query the `elasticsearch_test` container via: -``` -docker-compose exec elasticsearch_test curl http://localhost:9200 -``` + docker-compose exec elasticsearch_test curl http://localhost:9200 #### `PostgreSQL` @@ -247,25 +240,25 @@ Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via The system consists of several services that live in docker containers: -- `web` — the web interface on [http://localhost:5001](http://localhost:5001) -- `api` — the main server on [http://localhost:5000](http://localhost:5000) -- `api_test` — the api server for running tests via `prove` -- `elasticsearch` — database for `api` -- `elasticsearch_test` — database for `api_test` -- `pgdb` - PostgreSQL database container -- `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) -- `grep` - the web interface for grep.metacpan on [http://localhost:3001](http://localhost:3001) +* `web` — the web interface on [http://localhost:5001](http://localhost:5001) +* `api` — the main server on [http://localhost:5000](http://localhost:5000) +* `api_test` — the api server for running tests via `prove` +* `elasticsearch` — database for `api` +* `elasticsearch_test` — database for `api_test` +* `pgdb` - PostgreSQL database container +* `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) +* `grep` - the web interface for grep.metacpan on [http://localhost:3001](http://localhost:3001) These services use one or more Docker volumes: -- `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN` -- `metacpan_elasticsearch`: holds the Elasticsearch database files -- `metacpan_elasticsearch_test`: holds the Elasticsearch test database files -- `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies +* `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN` +* `metacpan_elasticsearch`: holds the Elasticsearch database files +* `metacpan_elasticsearch_test`: holds the Elasticsearch test database files +* `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies installed by [Carton][4] for the `api` and `web` services, respectively; mounted on `/carton` instead of `local`, to prevent clashing with the host user's Carton -- `metacpan_git_shared`: points to the git repo containing all extracted CPAN +* `metacpan_git_shared`: points to the git repo containing all extracted CPAN versions. This is mounted in `/shared/metacpan_git`. This can be either `metacpan-cpan-extracted` or `metacpan-cpan-extracted-lite`. The volume is bound to the local repo at `${PWD}/src/metacpan-cpan-extracted`. @@ -472,7 +465,7 @@ configuring e.g. a `kibana` service. If you run `docker ps` you'll see the containers. You might see something like: -``` +```bash $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 2efb9c475c83 metacpan-web:latest "carton exec plackup…" 12 hours ago Up 12 hours 0.0.0.0:5001->5001/tcp metacpan_web_1 From dd7cfa73ef838fe378d84629b24b2cd1418c2cf3 Mon Sep 17 00:00:00 2001 From: Alexander Hartmaier Date: Wed, 8 May 2024 17:43:34 +0200 Subject: [PATCH 14/45] fix metacpan-docker init when .git/hooks directory isn't created by git clone --- bin/metacpan-docker | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/metacpan-docker b/bin/metacpan-docker index d8ebd7a..c4cf5a8 100755 --- a/bin/metacpan-docker +++ b/bin/metacpan-docker @@ -16,6 +16,7 @@ git_clone_and_setup_hooks() { [ -d "$repo" ] || git clone "https://github.com/metacpan/$repo.git" cd "$repo" [ -e git/hooks/pre-commit ] && chmod +x git/hooks/pre-commit + [ -d .git/hooks ] || mkdir .git/hooks cd .git/hooks ln -sf ../../git/hooks/pre-commit ) From 9bf55def3b33cd119051e06e680f2a82a58ad254 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 1 Sep 2024 19:47:56 -0400 Subject: [PATCH 15/45] Tidy README.md --- README.md | 209 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 108 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 31cd866..61df61a 100644 --- a/README.md +++ b/README.md @@ -6,41 +6,41 @@ -* [Running the MetaCPAN stack with Docker (via Docker Compose)](#running-the-metacpan-stack-with-docker-via-docker-compose) -* [Quick Start](#quick-start) -* [Working with Containers](#working-with-containers) - * [Building Containers](#building-containers) - * [Accessing Containers](#accessing-containers) - * [Accessing Services](#accessing-services) - * [`web`](#web) - * [`api`](#api) - * [`Elasticsearch`](#elasticsearch) - * [`PostgreSQL`](#postgresql) - * [`grep`](#grep) -* [System architecture](#system-architecture) - * [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script) - * [`bin/metacpan init`](#binmetacpan-init) - * [`bin/metacpan localapi`](#binmetacpan-localapi) - * [`bin/metacpan-docker pull`](#binmetacpan-docker-pull) - * [`bin/metacpan-docker reset`](#binmetacpan-docker-reset) - * [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop) - * [Services](#services) - * [`web`](#web-1) - * [`api`](#api-1) - * [`grep`](#grep-1) - * [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service) - * [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices) - * [Putting the above all together](#putting-the-above-all-together) - * [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test) -* [Tips and tricks](#tips-and-tricks) - * [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker) - * [Running tests](#running-tests) - * [Updating Carton dependencies](#updating-carton-dependencies) - * [Updating the git repositories](#updating-the-git-repositories) - * [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data) -* [Peeking Inside the Container](#peeking-inside-the-container) -* [To Do](#to-do) -* [See also](#see-also) +- [Running the MetaCPAN stack with Docker (via Docker Compose)](#running-the-metacpan-stack-with-docker-via-docker-compose) +- [Quick Start](#quick-start) +- [Working with Containers](#working-with-containers) + - [Building Containers](#building-containers) + - [Accessing Containers](#accessing-containers) + - [Accessing Services](#accessing-services) + - [`web`](#web) + - [`api`](#api) + - [`Elasticsearch`](#elasticsearch) + - [`PostgreSQL`](#postgresql) + - [`grep`](#grep) +- [System architecture](#system-architecture) + - [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script) + - [`bin/metacpan init`](#binmetacpan-init) + - [`bin/metacpan localapi`](#binmetacpan-localapi) + - [`bin/metacpan-docker pull`](#binmetacpan-docker-pull) + - [`bin/metacpan-docker reset`](#binmetacpan-docker-reset) + - [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop) + - [Services](#services) + - [`web`](#web-1) + - [`api`](#api-1) + - [`grep`](#grep-1) + - [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service) + - [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices) + - [Putting the above all together](#putting-the-above-all-together) + - [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test) +- [Tips and tricks](#tips-and-tricks) + - [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker) + - [Running tests](#running-tests) + - [Updating Carton dependencies](#updating-carton-dependencies) + - [Updating the git repositories](#updating-the-git-repositories) + - [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data) +- [Peeking Inside the Container](#peeking-inside-the-container) +- [To Do](#to-do) +- [See also](#see-also) @@ -67,16 +67,16 @@ access docker follow the This document assumes the post-installation steps have been followed for the current user. -You will also need Docker buildx, and to enable Docker BuildKit. They should -be set up by default when using Docker Desktop, but on Linux you may need to +You will also need Docker buildx, and to enable Docker BuildKit. They should be +set up by default when using Docker Desktop, but on Linux you may need to install them. buildx is the `docker-buildx` package on Debian based systems. Docker BuildKit can be enabled by following the [Getting Started](https://docs.docker.com/build/buildkit/#getting-started) instructions. -If you are running a Mac ARM64 system, you will need to manually tell docker -to use the x86_64 version of Elasticsearch 2.4. This can be done by running -the command: +If you are running a Mac ARM64 system, you will need to manually tell docker to +use the x86_64 version of Elasticsearch 2.4. This can be done by running the +command: docker pull elasticsearch:2.4 --platform=linux/x86_64 @@ -92,10 +92,10 @@ Then, clone this repo and set up the environment: The `bin/metacpan-docker init` command clones the source repositories for: -* `metacpan-web` -* `metacpan-api` -* `metacpan-grep-front-end` -* `metacpan-cpan-extracted-lite` +- `metacpan-web` +- `metacpan-api` +- `metacpan-grep-front-end` +- `metacpan-cpan-extracted-lite` These repositories are automatically mounted into the appropriate docker containers allowing the developer to use their preferred tools to work with the @@ -104,8 +104,8 @@ source code. The `docker-compose up` command on its own will bring up the entire stack in the foreground (logs will be displayed). -The `docker-compose up` command will also fetch the official container images from -[MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) +The `docker-compose up` command will also fetch the official container images +from [MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) repositories. This will build the Docker containers for MetaCPAN, PostgreSQL and Elasticsearch @@ -118,7 +118,7 @@ following command in a separate terminal to get yourself up to speed: docker-compose exec api index-cpan.sh This will prompt you to confirm removing old indices and setting up mappings on -the Elasticsearch service (say `YES`). It will then proceed to rsync a partial +the Elasticsearch service (say `YES`). It will then proceed to rsync a partial CPAN in `/CPAN` for its metadata to be imported. Once the above is done, you should be able to see your local partial CPAN data @@ -147,8 +147,8 @@ For further details, read on! ### Building Containers -You can (re)build arbitrary containers. For instance, if you want to rebuild -the `api` container: +You can (re)build arbitrary containers. For instance, if you want to rebuild the +`api` container: docker-compose build api @@ -176,12 +176,12 @@ To access the `psql` command line client in the PostgreSQL container: Each container is responsible for a different service. Some of these services are available in the developer environment via ports on the host system. -We are using [traefik][13] to manage the traffic between services. -The current configuration is: +We are using [traefik][13] to manage the traffic between services. The current +configuration is: -* api: [http://api.metacpan.localhost](http://api.metacpan.localhost) -* web: [http://web.metacpan.localhost](http://web.metacpan.localhost) -* grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) +- api: [http://api.metacpan.localhost](http://api.metacpan.localhost) +- web: [http://web.metacpan.localhost](http://web.metacpan.localhost) +- grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) In order to access to the localhost subdomains, you probably have to manually add these entries in you `/etc/hosts` file. @@ -202,13 +202,13 @@ You can access the dashboard configuration via: The local instance of the web front end is accessible via: -* [http://localhost:5001](http://localhost:5001) -* [http://web.metacpan.localhost](http://web.metacpan.localhost) +- [http://localhost:5001](http://localhost:5001) +- [http://web.metacpan.localhost](http://web.metacpan.localhost) #### `api` -* [http://localhost:5000](http://localhost:5000) -* [http://api.metacpan.localhost](http://api.metacpan.localhost) +- [http://localhost:5000](http://localhost:5000) +- [http://api.metacpan.localhost](http://api.metacpan.localhost) #### `Elasticsearch` @@ -231,7 +231,7 @@ The PostgreSQL service by default is only accessible from other containers. The grep metacpan front end is accessible via: -* [http://grep.metacpan.localhost](http://grep.metacpan.localhost) +- [http://grep.metacpan.localhost](http://grep.metacpan.localhost) Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via [metacpan-cpan-extracted-lite](https://github.com/metacpan/metacpan-cpan-extracted-lite). @@ -240,28 +240,29 @@ Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via The system consists of several services that live in docker containers: -* `web` — the web interface on [http://localhost:5001](http://localhost:5001) -* `api` — the main server on [http://localhost:5000](http://localhost:5000) -* `api_test` — the api server for running tests via `prove` -* `elasticsearch` — database for `api` -* `elasticsearch_test` — database for `api_test` -* `pgdb` - PostgreSQL database container -* `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) -* `grep` - the web interface for grep.metacpan on [http://localhost:3001](http://localhost:3001) +- `web` — the web interface on [http://localhost:5001](http://localhost:5001) +- `api` — the main server on [http://localhost:5000](http://localhost:5000) +- `api_test` — the api server for running tests via `prove` +- `elasticsearch` — database for `api` +- `elasticsearch_test` — database for `api_test` +- `pgdb` - PostgreSQL database container +- `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io) +- `grep` - the web interface for grep.metacpan on + [http://localhost:3001](http://localhost:3001) These services use one or more Docker volumes: -* `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN` -* `metacpan_elasticsearch`: holds the Elasticsearch database files -* `metacpan_elasticsearch_test`: holds the Elasticsearch test database files -* `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies +- `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN` +- `metacpan_elasticsearch`: holds the Elasticsearch database files +- `metacpan_elasticsearch_test`: holds the Elasticsearch test database files +- `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies installed by [Carton][4] for the `api` and `web` services, respectively; mounted on `/carton` instead of `local`, to prevent clashing with the host user's Carton -* `metacpan_git_shared`: points to the git repo containing all extracted CPAN - versions. This is mounted in `/shared/metacpan_git`. - This can be either `metacpan-cpan-extracted` or `metacpan-cpan-extracted-lite`. - The volume is bound to the local repo at `${PWD}/src/metacpan-cpan-extracted`. +- `metacpan_git_shared`: points to the git repo containing all extracted CPAN + versions. This is mounted in `/shared/metacpan_git`. This can be either + `metacpan-cpan-extracted` or `metacpan-cpan-extracted-lite`. The volume is + bound to the local repo at `${PWD}/src/metacpan-cpan-extracted`. [4]: https://metacpan.org/pod/Carton @@ -283,7 +284,8 @@ The `init` subcommand basically clones the [metacpan-api][5] and them, in preparation for future `docker-compose` or `bin/metacpan-docker localapi` commands. -It also clones the `metacpan-grep-front-end` and `metacpan-cpan-extracted-lite` repositories. +It also clones the `metacpan-grep-front-end` and `metacpan-cpan-extracted-lite` +repositories. [5]: https://github.com/metacpan/metacpan-api [6]: https://github.com/metacpan/metacpan-web @@ -299,22 +301,22 @@ default `docker-compose.yml`. #### `bin/metacpan-docker pull` -This is used to update all the git repository in `src/*`. -This will stay on your current local branch. +This is used to update all the git repository in `src/*`. This will stay on your +current local branch. #### `bin/metacpan-docker reset` -This is used to reset all the git repositories in `src/*` to their -latest version on `upstream/master`. -This will fail if you have some uncommitted local changes. -You should then commit or cancel your changes before re-running the command. +This is used to reset all the git repositories in `src/*` to their latest +version on `upstream/master`. This will fail if you have some uncommitted local +changes. You should then commit or cancel your changes before re-running the +command. #### `bin/metacpan-docker` build/up/down/start/stop/run/ps/top... -As noted earlier, `bin/metacpan-docker` is a thin wrapper around `docker-compose`, -so commands like `up`, `down`, and `run` will work as expected from -`docker-compose`. See the [docker-compose docs][7] for an overview of available -commands. +As noted earlier, `bin/metacpan-docker` is a thin wrapper around +`docker-compose`, so commands like `up`, `down`, and `run` will work as expected +from `docker-compose`. See the [docker-compose docs][7] for an overview of +available commands. [7]: https://docs.docker.com/compose/reference/overview/#command-options-overview-and-help @@ -326,8 +328,9 @@ commands. The `web` service is a checkout of `metacpan-web`, built as a Docker image. Running this service alone is enough if you want to just hack on the frontend, since by default the service is configured to talk to -[https://fastapi.metacpan.org](https://fastapi.metacpan.org) for its backend; if this is what you want, then you -can simply invoke `docker-compose up` or `docker-compose up web`. +[https://fastapi.metacpan.org](https://fastapi.metacpan.org) for its backend; if +this is what you want, then you can simply invoke `docker-compose up` or +`docker-compose up web`. #### `api` @@ -340,8 +343,9 @@ additional commands in a separate terminal once #### `grep` -The `grep` service is a checkout of `metacpan-grep-front-end`, built as a Docker image. Note that this is using the `metacpan_git_shared` volume, which requires the git repo for -`metacpan-cpan-extracted` which can be initialized by running: +The `grep` service is a checkout of `metacpan-grep-front-end`, built as a Docker +image. Note that this is using the `metacpan_git_shared` volume, which requires +the git repo for `metacpan-cpan-extracted` which can be initialized by running: ./bin/metacpan-docker init @@ -351,9 +355,9 @@ Running bin/metacpan-docker localapi exec api partial-cpan-mirror.sh -will `rsync` modules from selected CPAN authors, plus the package and author indices, -into the `api` service's `/CPAN` directory. This is nearly equivalent to the -same script in the (now deprecated) [metacpan-developer][8] repository. +will `rsync` modules from selected CPAN authors, plus the package and author +indices, into the `api` service's `/CPAN` directory. This is nearly equivalent +to the same script in the (now deprecated) [metacpan-developer][8] repository. [8]: https://github.com/metacpan/metacpan-developer ##### Bootstrapping the `elasticsearch` indices @@ -380,7 +384,9 @@ instead will set it all up for you. The `elasticsearch` and `elasticsearch_test` services use the official [Elasticsearch Docker image][9], configured with settings and scripts taken from -the [metacpan-puppet][10] repository. The `api` service depends on the `elasticsearch` service and the `api_test` service depends on the `elasticsearch_test` services. +the [metacpan-puppet][10] repository. The `api` service depends on the +`elasticsearch` service and the `api_test` service depends on the +`elasticsearch_test` services. [9]: https://store.docker.com/images/elasticsearch [10]: https://github.com/metacpan/metacpan-puppet @@ -445,16 +451,17 @@ You can use `bin/metacpan-docker pull` to update all `src/*` directories. ### Running Kibana to peek into Elasticsearch data By default, the `docker-compose.localapi.yml` configures the `elasticsearch` -service to listen on the Docker host at [http://localhost:9200](http://localhost:9200), and is also -accessible via the Docker `default` network address of [http://172.17.0.1:9200](http://172.17.0.1:9200); -you can inspect it via simple `curl` or `wget` requests, or use a [Kibana][12] -container, e.g. +service to listen on the Docker host at +[http://localhost:9200](http://localhost:9200), and is also accessible via the +Docker `default` network address of +[http://172.17.0.1:9200](http://172.17.0.1:9200); you can inspect it via simple +`curl` or `wget` requests, or use a [Kibana][12] container, e.g. docker run --rm -p 5601:5601 -e ELASTICSEARCH_URL=http://172.17.0.1:9200 -it kibana:4.6 -Running the above will provide a Kibana container at [http://localhost:5601](http://localhost:5601), -which you can configure to have it read the `cpan*` index in the `elasticsearch` -service. +Running the above will provide a Kibana container at +[http://localhost:5601](http://localhost:5601), which you can configure to have +it read the `cpan*` index in the `elasticsearch` service. It is also certainly possible to run Kibana as part of the compose setup, by configuring e.g. a `kibana` service. From 0a8ba6f44982b0ebe815e0eb3040a6907fb0f609 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 1 Sep 2024 19:54:36 -0400 Subject: [PATCH 16/45] version in docker-compose.yml is now obsolete --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a47f53..5f1d52f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "3.4" # ____ _____ ______ _____ ____ _____ ____ # / ___|| ____| _ \ \ / /_ _/ ___| ____/ ___| From e8c4a8142cf0cb70bc4ed9b9f9c68df174cd1f93 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 1 Sep 2024 20:18:25 -0400 Subject: [PATCH 17/45] s/docker-compose/docker compose/ in bin/metacpan-docker --- bin/metacpan-docker | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/metacpan-docker b/bin/metacpan-docker index c4cf5a8..01d98ed 100755 --- a/bin/metacpan-docker +++ b/bin/metacpan-docker @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# metacpan-docker: simple wrapper for docker-compose running MetaCPAN +# metacpan-docker: simple wrapper for docker compose running MetaCPAN set -e @@ -7,7 +7,6 @@ GitRepos=("metacpan-api" "metacpan-web" "metacpan-grep-front-end" "metacpan-cpan # sanity check type "docker" > /dev/null -type "docker-compose" > /dev/null git_clone_and_setup_hooks() { local repo=$1 @@ -108,4 +107,4 @@ case "x$1" in ;; esac -exec docker-compose "$@" +exec docker compose "$@" From b3990721d926260aee6ac740fd0bb19b6ab39c9c Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 1 Sep 2024 20:19:40 -0400 Subject: [PATCH 18/45] Tidy bin/metacpan-docker --- bin/metacpan-docker | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/bin/metacpan-docker b/bin/metacpan-docker index 01d98ed..139b4b4 100755 --- a/bin/metacpan-docker +++ b/bin/metacpan-docker @@ -81,30 +81,28 @@ reset_repo() { done } - case "x$1" in - 'xinit') - init - exit - ;; - 'xreset') - reset_repo - exit - ;; - 'xpull') - update - exit - ;; - 'xlocalapi') - shift - ;; - 'x') - init - update - exit - ;; - *) - ;; +'xinit') + init + exit + ;; +'xreset') + reset_repo + exit + ;; +'xpull') + update + exit + ;; +'xlocalapi') + shift + ;; +'x') + init + update + exit + ;; +*) ;; esac exec docker compose "$@" From 4bb6cc2a881c5e0bffbae7b9c79dd9c1d051c1e9 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 1 Sep 2024 20:25:50 -0400 Subject: [PATCH 19/45] s/docker-compose/docker compose/ in README --- README.md | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 61df61a..6d6daee 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CircleCI](https://circleci.com/gh/metacpan/metacpan-docker.svg?style=svg)](https://circleci.com/gh/metacpan/metacpan-docker) -![docker-compose up](https://github.com/metacpan/metacpan-docker/workflows/docker-compose%20up/badge.svg?branch=master) +![docker compose up](https://github.com/metacpan/metacpan-docker/workflows/docker-compose%20up/badge.svg?branch=master) @@ -80,8 +80,8 @@ command: docker pull elasticsearch:2.4 --platform=linux/x86_64 -It is highly recommended that you alias `docker-compose` to `fig` (its original -name) and use it wherever `docker-compose` is used. You are going to have to +It is highly recommended that you alias `docker compose` to `fig` (its original +name) and use it wherever `docker compose` is used. You are going to have to type this command a lot. Then, clone this repo and set up the environment: @@ -101,10 +101,10 @@ These repositories are automatically mounted into the appropriate docker containers allowing the developer to use their preferred tools to work with the source code. -The `docker-compose up` command on its own will bring up the entire stack in the +The `docker compose up` command on its own will bring up the entire stack in the foreground (logs will be displayed). -The `docker-compose up` command will also fetch the official container images +The `docker compose up` command will also fetch the official container images from [MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) repositories. @@ -115,7 +115,7 @@ Docker) and run the services. Don't forget to seed the local `metacpan-api` with a partial CPAN; run the following command in a separate terminal to get yourself up to speed: - docker-compose exec api index-cpan.sh + docker compose exec api index-cpan.sh This will prompt you to confirm removing old indices and setting up mappings on the Elasticsearch service (say `YES`). It will then proceed to rsync a partial @@ -128,7 +128,7 @@ elsewhere. Alternatively, if you just want to hack on the web frontend, you can run this instead of all the above: - docker-compose up web + docker compose up web From here, you can proceed and hack on the MetaCPAN code at `src/metacpan-api` and/or `src/metacpan-web` directories, and saving edits will reload the @@ -137,7 +137,7 @@ corresponding apps automatically! When done hacking (or, more likely, when you need to rebuild/refresh your Docker environment) you can then run - docker-compose down + docker compose down in another terminal to stop all MetaCPAN services and remove the containers. @@ -150,26 +150,26 @@ For further details, read on! You can (re)build arbitrary containers. For instance, if you want to rebuild the `api` container: - docker-compose build api + docker compose build api ### Accessing Containers -Containers are accessible via the `docker-compose exec` command followed by the +Containers are accessible via the `docker compose exec` command followed by the container and then the command to execute. For example, to start a shell prompt in the `api` container: - docker-compose exec api /bin/bash + docker compose exec api /bin/bash Executing tests via `prove` inside the API container: - docker-compose exec api_test prove -lvr \ + docker compose exec api_test prove -lvr \ t/00_setup.t \ t/01_darkpan.t \ t/api/controller/cover.t To access the `psql` command line client in the PostgreSQL container: - docker-compose exec pgdb psql + docker compose exec pgdb psql ### Accessing Services @@ -217,11 +217,11 @@ directly. They are available via the `api` and `api_test` containers. You can query the `elasticsearch` container via: - docker-compose exec elasticsearch curl http://localhost:9200 + docker compose exec elasticsearch curl http://localhost:9200 You can query the `elasticsearch_test` container via: - docker-compose exec elasticsearch_test curl http://localhost:9200 + docker compose exec elasticsearch_test curl http://localhost:9200 #### `PostgreSQL` @@ -267,13 +267,13 @@ These services use one or more Docker volumes: [4]: https://metacpan.org/pod/Carton Docker Compose is used to, uh, _compose_ them all together into one system. -Using `docker-compose` directly is a mouthful, however, so putting this all +Using `docker compose` directly is a mouthful, however, so putting this all together is done via the `bin/metacpan-docker` script to simplify setup and usage (and to get you started hacking on the MetaCPAN sooner!) ### The `bin/metacpan-docker` script -`bin/metacpan-docker` is a thin wrapper around the `docker-compose` command, +`bin/metacpan-docker` is a thin wrapper around the `docker compose` command, providing the environment variables necessary to run a basic MetaCPAN environment. It provides these subcommands: @@ -281,7 +281,7 @@ environment. It provides these subcommands: The `init` subcommand basically clones the [metacpan-api][5] and [metacpan-web][6] repositories, and sets up the git commit hooks for each of -them, in preparation for future `docker-compose` or +them, in preparation for future `docker compose` or `bin/metacpan-docker localapi` commands. It also clones the `metacpan-grep-front-end` and `metacpan-cpan-extracted-lite` @@ -292,12 +292,12 @@ repositories. #### `bin/metacpan localapi` -The `localapi` subcommand adds the necessary configuration for `docker-compose` +The `localapi` subcommand adds the necessary configuration for `docker compose` to run both the `metacpan-web` and `metacpan-api` services, along with `elasticsearch` and Docker volumes. Under the hood, it customizes the `COMPOSE_FILE` and `COMPOSE_PROJECT_NAME` environment variables used by -`docker-compose` to use additional YAML configuration files aside from the -default `docker-compose.yml`. +`docker compose` to use additional YAML configuration files aside from the +default `docker compose.yml`. #### `bin/metacpan-docker pull` @@ -314,8 +314,8 @@ command. #### `bin/metacpan-docker` build/up/down/start/stop/run/ps/top... As noted earlier, `bin/metacpan-docker` is a thin wrapper around -`docker-compose`, so commands like `up`, `down`, and `run` will work as expected -from `docker-compose`. See the [docker-compose docs][7] for an overview of +`docker compose`, so commands like `up`, `down`, and `run` will work as expected +from `docker compose`. See the [docker compose docs][7] for an overview of available commands. [7]: @@ -329,8 +329,8 @@ The `web` service is a checkout of `metacpan-web`, built as a Docker image. Running this service alone is enough if you want to just hack on the frontend, since by default the service is configured to talk to [https://fastapi.metacpan.org](https://fastapi.metacpan.org) for its backend; if -this is what you want, then you can simply invoke `docker-compose up` or -`docker-compose up web`. +this is what you want, then you can simply invoke `docker compose up` or +`docker compose up web`. #### `api` @@ -396,7 +396,7 @@ the [metacpan-puppet][10] repository. The `api` service depends on the ### Running your own miniCPAN inside metacpan-docker Suppose you have a local minicpan in `/home/ftp/pub/CPAN`. If you would like to -use this in `metacpan-docker`, then edit the `docker-compose.localapi.yml` to +use this in `metacpan-docker`, then edit the `docker compose.localapi.yml` to change the `api` service's volume mounts to use your local minicpan as `/CPAN`, e.g.: @@ -450,7 +450,7 @@ You can use `bin/metacpan-docker pull` to update all `src/*` directories. ### Running Kibana to peek into Elasticsearch data -By default, the `docker-compose.localapi.yml` configures the `elasticsearch` +By default, the `docker compose.localapi.yml` configures the `elasticsearch` service to listen on the Docker host at [http://localhost:9200](http://localhost:9200), and is also accessible via the Docker `default` network address of From 028feef1bba47cfb61ac04fa82a02119d8fb35a3 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Mon, 9 Sep 2024 22:27:47 -0400 Subject: [PATCH 20/45] Updat econfigs/metacpan-api/metacpan_server_testing.conf --- configs/metacpan-api/metacpan_server_testing.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configs/metacpan-api/metacpan_server_testing.conf b/configs/metacpan-api/metacpan_server_testing.conf index 1600860..396826d 100644 --- a/configs/metacpan-api/metacpan_server_testing.conf +++ b/configs/metacpan-api/metacpan_server_testing.conf @@ -1,6 +1,15 @@ cpan var/t/tmp/fakecpan +die_on_error 1 +level warn source_base var/t/tmp/source +elasticsearch_servers = elasticsearch_test:9200 + + + class Log::Log4perl::Appender::Screen + name testing + + servers __ENV(ES)__ From 74f5ddbffeb8dd320c74e0d034ef7cbc7fe0fe23 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 19 Sep 2024 11:21:01 -0400 Subject: [PATCH 21/45] Remove api and web configs Use the configs in the respective repos. I'm not sure why this was done, but this setup makes it very easy for the configs to get out of sync. Since we are not using docker compose in production, I don't think we should make the config file setup any more complicated it needs to be. --- configs/metacpan-api/metacpan.pl | 23 ----------- configs/metacpan-api/metacpan_server.conf | 25 ------------ .../metacpan-api/metacpan_server_testing.conf | 40 ------------------- configs/metacpan-web/metacpan_web_local.conf | 8 ---- docker-compose.yml | 28 ------------- 5 files changed, 124 deletions(-) delete mode 100644 configs/metacpan-api/metacpan.pl delete mode 100644 configs/metacpan-api/metacpan_server.conf delete mode 100644 configs/metacpan-api/metacpan_server_testing.conf delete mode 100644 configs/metacpan-web/metacpan_web_local.conf diff --git a/configs/metacpan-api/metacpan.pl b/configs/metacpan-api/metacpan.pl deleted file mode 100644 index 8821e2e..0000000 --- a/configs/metacpan-api/metacpan.pl +++ /dev/null @@ -1,23 +0,0 @@ -# do not edit this file -# create etc/metacpan_local.pl instead -use FindBin; - -{ - # ElasticSearch instance, can be either a single server - # or an arrayref of servers - es => 'elasticsearch:9200', - # the port of the api server - port => '5000', - # log level - level => 'info', - # appender for Log4perl - # default layout is "%d %p{1} %c: %m{chomp}%n" - # can be overridden using the layout key - # defining logger in metacpan_local.pl will - # override and not append to this configuration - logger => [{ - class => 'Log::Log4perl::Appender::File', - filename => $FindBin::RealBin . '/../var/log/metacpan.log', - syswrite => 1, - }] -} diff --git a/configs/metacpan-api/metacpan_server.conf b/configs/metacpan-api/metacpan_server.conf deleted file mode 100644 index 1894643..0000000 --- a/configs/metacpan-api/metacpan_server.conf +++ /dev/null @@ -1,25 +0,0 @@ -git /usr/bin/git -cpan /CPAN -minion_dsn = postgresql://metacpan:metacpan@pgdb/minion_queue -secret = I wish I had one to keep - - - servers elasticsearch:9200 - - - - servers elasticsearch:9200 - - - - servers elasticsearch:9200 - - - - servers elasticsearch:9200 - - - - # required for server startup -- override this in metacpan_server_local.conf - private_key 59125ffc09413eed3f2a2c07a37c7a44b95633e2 - diff --git a/configs/metacpan-api/metacpan_server_testing.conf b/configs/metacpan-api/metacpan_server_testing.conf deleted file mode 100644 index 396826d..0000000 --- a/configs/metacpan-api/metacpan_server_testing.conf +++ /dev/null @@ -1,40 +0,0 @@ -cpan var/t/tmp/fakecpan -die_on_error 1 -level warn -source_base var/t/tmp/source - -elasticsearch_servers = elasticsearch_test:9200 - - - class Log::Log4perl::Appender::Screen - name testing - - - - servers __ENV(ES)__ - - - - servers __ENV(ES)__ - - - - servers __ENV(ES)__ - - - - captcha_class Captcha::Mock - private_key testing - - -github_key = foo -github_secret = bar - -secret weak - - - host smtp.fastmail.com - port 465 - username foo@metacpan.org - password seekrit - diff --git a/configs/metacpan-web/metacpan_web_local.conf b/configs/metacpan-web/metacpan_web_local.conf deleted file mode 100644 index 16dc12c..0000000 --- a/configs/metacpan-web/metacpan_web_local.conf +++ /dev/null @@ -1,8 +0,0 @@ -api = http://api:5000 -api_public = http://localhost:5000 -source_host = http://localhost:5000 -web_host = http://localhost:5001 - - - cache_dir = /var/tmp/templates - diff --git a/docker-compose.yml b/docker-compose.yml index 5f1d52f..86ae5e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,10 +74,6 @@ services: source: ./src/metacpan-web target: /metacpan-web # read_only: true - - type: bind - source: ./configs/metacpan-web/metacpan_web_local.conf - target: /metacpan-web/metacpan_web_local.conf - read_only: true ports: - "5001:5001" networks: @@ -118,18 +114,6 @@ services: - type: bind source: ./src/metacpan-api target: /metacpan-api - - type: bind - source: ./configs/metacpan-api/metacpan_server.conf - target: /metacpan-api/metacpan_server.conf - read_only: true - - type: bind - source: ./configs/metacpan-api/metacpan_server_testing.conf - target: /metacpan-api/metacpan_server_testing.conf - read_only: true - - type: bind - source: ./configs/metacpan-api/metacpan.pl - target: /metacpan-api/etc/metacpan.pl - read_only: true - type: bind source: ./bin/index-cpan.sh target: /bin/index-cpan.sh @@ -171,18 +155,6 @@ services: - type: bind source: ./src/metacpan-api target: /metacpan-api - - type: bind - source: ./configs/metacpan-api/metacpan_server.conf - target: /metacpan-api/metacpan_server.conf - read_only: true - - type: bind - source: ./configs/metacpan-api/metacpan_server_testing.conf - target: /metacpan-api/metacpan_server_testing.conf - read_only: true - - type: bind - source: ./configs/metacpan-api/metacpan.pl - target: /metacpan-api/etc/metacpan.pl - read_only: true - type: bind source: ./bin/index-cpan.sh target: /bin/index-cpan.sh From 6a34c33f0d17a68698b370eeab779aa5df3e28c9 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 21 Sep 2024 18:34:49 -0400 Subject: [PATCH 22/45] Consolidate env vars + fix minion under compose Since we're not deploying this, we can try to set environment variables in fewer places. --- .env | 17 +++++++++++--- docker-compose.yml | 25 ++++++++++++--------- localapi.env | 9 -------- pg/docker-entrypoint-initdb.d/100-roles.sql | 2 +- 4 files changed, 29 insertions(+), 24 deletions(-) delete mode 100644 localapi.env diff --git a/.env b/.env index 074398d..854abd3 100644 --- a/.env +++ b/.env @@ -1,4 +1,15 @@ -COMPOSE_PROJECT_NAME=metacpan +API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose" +COLUMNS=80 +ES=elasticsearch:9200 +ES_TEST=elasticsearch_test:9200 +MINICPAN=/CPAN +MOJO_MODE=development +NET_ASYNC_HTTP_MAXCONNS=1 +PERL_CARTON_PATH=/carton +PERL_MM_USE_DEFAULT=1 +PG_DATABASE=metacpan +PG_HOST=pghost +PG_PASSWORD=t00lchain +PG_PORT=5432 +PG_USER=metacpan PLACK_ENV=development -PGDB=pgdb:5432 -API_SERVER=morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose diff --git a/docker-compose.yml b/docker-compose.yml index 5f1d52f..9d961a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ --- - +x-project: + name: metacpan # ____ _____ ______ _____ ____ _____ ____ # / ___|| ____| _ \ \ / /_ _/ ___| ____/ ___| # \___ \| _| | |_) \ \ / / | | | | _| \___ \ @@ -100,16 +101,18 @@ services: api: depends_on: - elasticsearch - - pgdb + - pghost - traefik image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api + # put variables for compose inside a .env file + # use env_file for variables to be set inside the container env_file: - - localapi.env + - .env command: > /metacpan-api/wait-for-es.sh http://elasticsearch:9200 "" -- - /metacpan-api/wait-for-it.sh -t 15 -s ${PGDB} -- + /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -154,7 +157,7 @@ services: api_test: depends_on: - elasticsearch_test - - pgdb + - pghost image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api @@ -162,7 +165,7 @@ services: - localapi_test.env command: > /metacpan-api/wait-for-es.sh http://elasticsearch_test:9200 "" -- - /metacpan-api/wait-for-it.sh -t 15 -s ${PGDB} -- + /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -315,8 +318,8 @@ services: # |_| |___/ |_| # - pgdb: - hostname: pgdb + pghost: + hostname: pghost image: "postgres:${PG_VERSION_TAG:-9.6-alpine}" build: context: "./pg" @@ -324,7 +327,7 @@ services: PG_TAG: "${PG_VERSION_TAG:-9.6-alpine}" environment: POSTGRES_PASSWORD: metacpan - POSTGRES_USERNAME: metacpan123 + POSTGRES_USER: metacpan123 POSTGRES_DB: metacpan networks: - database @@ -336,7 +339,7 @@ services: test: ["CMD", "/healthcheck.sh"] volumes: - type: volume - source: pgdb-data + source: pghost-data target: /var/lib/postgresql/data - type: bind source: ./pg/docker-entrypoint-initdb.d @@ -373,6 +376,6 @@ volumes: cpan: elasticsearch: elasticsearch_test: - pgdb-data: + pghost-data: metacpan_git_shared: external: true diff --git a/localapi.env b/localapi.env deleted file mode 100644 index 97514a8..0000000 --- a/localapi.env +++ /dev/null @@ -1,9 +0,0 @@ -NET_ASYNC_HTTP_MAXCONNS=1 -COLUMNS=80 -ES=elasticsearch:9200 -ES_TEST=elasticsearch_test:9200 -ES_TEST=elasticsearch_test:9200 -MINICPAN=/CPAN -PERL_MM_USE_DEFAULT=1 -PERL_CARTON_PATH=/carton -MOJO_MODE=development diff --git a/pg/docker-entrypoint-initdb.d/100-roles.sql b/pg/docker-entrypoint-initdb.d/100-roles.sql index 6094210..867939c 100644 --- a/pg/docker-entrypoint-initdb.d/100-roles.sql +++ b/pg/docker-entrypoint-initdb.d/100-roles.sql @@ -1,4 +1,4 @@ -CREATE ROLE metacpan WITH LOGIN PASSWORD 'metacpan'; +CREATE ROLE metacpan WITH LOGIN PASSWORD 't00lchain'; CREATE ROLE "metacpan-api" WITH LOGIN; -- make things easier for when we're poking around from inside the container From 6b776cccc16e5259db6d1f94d2034cb8763a1e6d Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Mon, 23 Sep 2024 20:51:45 -0400 Subject: [PATCH 23/45] Remove traefik This is adding an extra (unneeded) layer of complexity to the dev environment. It's simpler to use ports than having to add hostnames to your system config and also having to troubleshoot traefik. --- docker-compose.yml | 50 ---------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 37af12b..0f3c2fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,32 +9,6 @@ x-project: # services: - # __ _____ __ - # / /__________ ____ / __(_) /__ - # / __/ ___/ __ `/ _ \/ /_/ / //_/ __ - # / /_/ / / /_/ / __/ __/ / ,< _| =\__ - # \__/_/ \__,_/\___/_/ /_/_/|_| /o____o_\ - - traefik: - # The official v2.4.5 Traefik docker image - image: traefik:v2.4.5 - networks: - - traefik-network - # Enables the web UI and tells Traefik to listen to docker - command: - - "--api.insecure=true" - - "--providers.docker" - # Do not expose containers unless explicitly told so - - "--providers.docker.exposedbydefault=false" - ports: - # The HTTP port - - "80:80" - # The Web UI (enabled by --api.insecure=true) - - "8080:8080" - volumes: - # So that Traefik can listen to the Docker events - - /var/run/docker.sock:/var/run/docker.sock - # _ _ # | | ___ __ _ ___ _ __ ___ _ _| |_ # | |/ _ \ / _` / __| '_ \ / _ \| | | | __| @@ -62,8 +36,6 @@ services: # web: - depends_on: - - traefik image: metacpan/metacpan-web:latest build: context: ./src/metacpan-web @@ -79,12 +51,6 @@ services: - "5001:5001" networks: - web-network - - traefik-network - labels: - - "traefik.enable=true" - - "traefik.docker.network=traefik-network" - - "traefik.http.routers.web.rule=Host(`web.metacpan.localhost`)" - - "traefik.http.services.web.loadbalancer.server.port=5001" # _ # __ _ _ __ (_) @@ -98,7 +64,6 @@ services: depends_on: - elasticsearch - pghost - - traefik image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api @@ -130,13 +95,7 @@ services: networks: - database - elasticsearch - - traefik-network - web-network - labels: - - "traefik.enable=true" - - "traefik.docker.network=traefik-network" - - "traefik.http.routers.api.rule=Host(`api.metacpan.localhost`)" - - "traefik.http.services.api.loadbalancer.server.port=5000" api_test: depends_on: @@ -195,8 +154,6 @@ services: # |___/ |_| grep: - depends_on: - - traefik image: metacpan/metacpan-grep-front-end:latest build: context: ./src/metacpan-grep-front-end @@ -211,12 +168,6 @@ services: read_only: true env_file: - grep.env - networks: - - traefik-network - labels: - - "traefik.enable=true" - - "traefik.http.routers.grep.rule=Host(`grep.metacpan.localhost`)" - - "traefik.http.services.grep-web.loadbalancer.server.port=3000" # ____ _ _____ _ ____ _ ____ _____ ____ # | _ \ / \|_ _|/ \ | __ ) / \ / ___|| ____/ ___| @@ -332,7 +283,6 @@ services: networks: database: elasticsearch: - traefik-network: web-network: # __ _____ _ _ _ __ __ _____ ____ From 9bfddb42b3482a63558559faf3e3928516bceef4 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Mon, 7 Oct 2024 22:36:59 -0400 Subject: [PATCH 24/45] web now exposes port 80 rather than 5001 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0f3c2fd..9964b39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: target: /metacpan-web # read_only: true ports: - - "5001:5001" + - "5001:80" networks: - web-network From c356cc6fd1389e9ba5fe2e15fa647276f0af75d8 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Mon, 7 Oct 2024 22:49:31 -0400 Subject: [PATCH 25/45] Get debug logging in web container by default --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9964b39..97d970c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,8 @@ services: - "5001:80" networks: - web-network + environment: + - PLACK_ENV=development # _ # __ _ _ __ (_) From 963f35260bb21c347064984f19f566441d0386ca Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 10 Oct 2024 11:03:24 +0200 Subject: [PATCH 26/45] include web docker-compose file Rather than having an entirely separate configuration for metacpan-web, include its docker-compose.yml file, but with some overrides. --- compose/web.yml | 6 ++++++ docker-compose.yml | 32 +++++--------------------------- 2 files changed, 11 insertions(+), 27 deletions(-) create mode 100644 compose/web.yml diff --git a/compose/web.yml b/compose/web.yml new file mode 100644 index 0000000..f85588b --- /dev/null +++ b/compose/web.yml @@ -0,0 +1,6 @@ +services: + web-server: + ports: + - "5001:80" + networks: + - web-network diff --git a/docker-compose.yml b/docker-compose.yml index 97d970c..1e14c98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,11 @@ x-project: # |____/|_____|_| \_\ \_/ |___\____|_____|____/ # +include: + - path: + - src/metacpan-web/docker-compose.yml + - compose/web.yml + services: # _ _ # | | ___ __ _ ___ _ __ ___ _ _| |_ @@ -28,32 +33,6 @@ services: ports: - "8100:80" - # _ - # __ _____| |__ - # \ \ /\ / / _ \ '_ \ - # \ V V / __/ |_) | - # \_/\_/ \___|_.__/ - # - - web: - image: metacpan/metacpan-web:latest - build: - context: ./src/metacpan-web - volumes: - - type: volume - source: web_carton - target: /carton - - type: bind - source: ./src/metacpan-web - target: /metacpan-web - # read_only: true - ports: - - "5001:80" - networks: - - web-network - environment: - - PLACK_ENV=development - # _ # __ _ _ __ (_) # / _` | '_ \| | @@ -295,7 +274,6 @@ networks: # volumes: - web_carton: api_carton: cpan: elasticsearch: From 6cd73139cc3da4aa6e13c2bae0ef7e4b0b121a34 Mon Sep 17 00:00:00 2001 From: Cesar Tessarin Date: Fri, 11 Oct 2024 15:40:08 -0300 Subject: [PATCH 27/45] Enable prompts from IO::Prompt::Tiny When the environment variable PERL_MM_USE_DEFAULT is set to true, all code using IO::Prompt::Tiny becomes non-functional. This specifically affects two locations: - metacpan-api: MetaCPAN::Role::Script::are_you_sure - metacpan-ingest: MetaCPAN::Ingest::are_you_sure Both prompt users for confirmation before critical operations. They set no default answer and perform a string equality test to 'YES' to proceed. Without a default value, PERL_MM_USE_DEFAULT causes these prompts to return an empty string, failing the confirmation check and rendering the prompt useless. This commit resolves the issue. --- .env | 1 - localapi_test.env | 1 - 2 files changed, 2 deletions(-) diff --git a/.env b/.env index 854abd3..6536994 100644 --- a/.env +++ b/.env @@ -6,7 +6,6 @@ MINICPAN=/CPAN MOJO_MODE=development NET_ASYNC_HTTP_MAXCONNS=1 PERL_CARTON_PATH=/carton -PERL_MM_USE_DEFAULT=1 PG_DATABASE=metacpan PG_HOST=pghost PG_PASSWORD=t00lchain diff --git a/localapi_test.env b/localapi_test.env index 1f49954..e899080 100644 --- a/localapi_test.env +++ b/localapi_test.env @@ -6,6 +6,5 @@ HARNESS_ACTIVE=1 # Instantiate Catalyst models using metacpan_server_testing.conf METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing MINICPAN=/CPAN -PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton MOJO_MODE=testing From 0f907dfcd60ce2804ad25326b5eff235ead5c84a Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 15 Oct 2024 19:23:06 -0400 Subject: [PATCH 28/45] specify image name for web-server Fixes "docker compose pull" --- compose/web.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/web.yml b/compose/web.yml index f85588b..445b0bb 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -1,5 +1,6 @@ services: web-server: + image: metacpan/metacpan-web:latest ports: - "5001:80" networks: From 1bea9753c585f3b772887cfd7f9d210634c852f1 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 15 Oct 2024 19:52:27 -0400 Subject: [PATCH 29/45] Remove logspout This seems to be configured to send logs to Honeycomb.io in production, but we aren't going to be deploying via compose. --- docker-compose.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e14c98..6a35324 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,25 +14,6 @@ include: - compose/web.yml services: - # _ _ - # | | ___ __ _ ___ _ __ ___ _ _| |_ - # | |/ _ \ / _` / __| '_ \ / _ \| | | | __| - # | | (_) | (_| \__ \ |_) | (_) | |_| | |_ - # |_|\___/ \__, |___/ .__/ \___/ \__,_|\__| - # |___/ |_| - # - - logspout: - image: honeycombio/logspout-honeycomb:1.13 - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - env_file: - - logging.env - ports: - - "8100:80" - # _ # __ _ _ __ (_) # / _` | '_ \| | From eabc6b9cabacde9d1a46d3a3e4ded65261c54dea Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 15 Oct 2024 19:53:56 -0400 Subject: [PATCH 30/45] tidy Docker config --- compose/web.yml | 1 + docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/web.yml b/compose/web.yml index 445b0bb..1f490a9 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -1,3 +1,4 @@ +--- services: web-server: image: metacpan/metacpan-web:latest diff --git a/docker-compose.yml b/docker-compose.yml index 6a35324..fa670f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ x-project: include: - path: - - src/metacpan-web/docker-compose.yml - - compose/web.yml + - src/metacpan-web/docker-compose.yml + - compose/web.yml services: # _ From b55126b3e2222c527d40b02e3847e51180776bb1 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 15 Oct 2024 20:45:12 -0400 Subject: [PATCH 31/45] Use compose profiles This allows us to do things like: docker compose --profile test up docker compose --profile dev up --- .circleci/config.yml | 23 ++++++++--------------- README.md | 13 +++++++------ docker-compose.yml | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a92c0d..b933d65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,6 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference +--- +# Use the latest 2.1 version of CircleCI pipeline process engine. See: +# https://circleci.com/docs/2.0/configuration-reference version: 2.1 # Orchestrate or schedule a set of jobs workflows: @@ -8,17 +10,8 @@ workflows: jobs: build-and-test: machine: true - resource_class: medium + resource_class: large steps: - # CircleCI has its own docker-compose already installed, but as of this writing, their version is too old to understand our docker-compose.yml - - run: - name: Install Docker Compose - command: | - set -x - curl -L https://github.com/docker/compose/releases/download/1.26.2/docker-compose-`uname -s`-`uname -m` > /home/circleci/bin/docker-compose - sudo chmod +x /home/circleci/bin/docker-compose - which docker-compose - docker-compose --version - checkout - run: command: | @@ -26,7 +19,7 @@ jobs: name: clone missing repositories - run: command: | - docker-compose --verbose up -d api_test + docker-compose --verbose --profile test up -d name: compose up # Since we're running docker-compose -d, we don't actually know if # Elasticsearch is available at the time this build step begins. We @@ -38,12 +31,12 @@ jobs: - run: name: Run complete MetaCPAN API Test Suite command: | - docker-compose exec -T api_test prove -lr --jobs 2 t - docker-compose down + docker-compose --profile test exec -T api_test prove -lr --jobs 4 t + docker-compose --profile test down - run: name: Show Docker container logs on Error command: | - docker-compose logs + docker-compose --profile test logs docker stats --no-stream docker ps -a | head when: on_fail diff --git a/README.md b/README.md index 6d6daee..2363ef6 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,12 @@ These repositories are automatically mounted into the appropriate docker containers allowing the developer to use their preferred tools to work with the source code. -The `docker compose up` command on its own will bring up the entire stack in the -foreground (logs will be displayed). +The `docker compose --profile dev up` command on its own will bring up the +entire stack in the foreground (logs will be displayed). -The `docker compose up` command will also fetch the official container images -from [MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) +The `docker compose --profile dev up` command will also fetch the official +container images from +[MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list) repositories. This will build the Docker containers for MetaCPAN, PostgreSQL and Elasticsearch @@ -128,7 +129,7 @@ elsewhere. Alternatively, if you just want to hack on the web frontend, you can run this instead of all the above: - docker compose up web + docker compose up web-server From here, you can proceed and hack on the MetaCPAN code at `src/metacpan-api` and/or `src/metacpan-web` directories, and saving edits will reload the @@ -137,7 +138,7 @@ corresponding apps automatically! When done hacking (or, more likely, when you need to rebuild/refresh your Docker environment) you can then run - docker compose down + docker compose --profile dev down in another terminal to stop all MetaCPAN services and remove the containers. diff --git a/docker-compose.yml b/docker-compose.yml index fa670f2..b0bb40f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,8 @@ services: # api: + profiles: + - dev depends_on: - elasticsearch - pghost @@ -60,6 +62,8 @@ services: - web-network api_test: + profiles: + - test depends_on: - elasticsearch_test - pghost @@ -94,6 +98,9 @@ services: - elasticsearch ingest: + profiles: + - dev + - ingest image: metacpan/metacpan-ingest:latest volumes: - type: volume @@ -116,6 +123,8 @@ services: # |___/ |_| grep: + profiles: + - grep image: metacpan/metacpan-grep-front-end:latest build: context: ./src/metacpan-grep-front-end @@ -146,6 +155,9 @@ services: # elasticsearch: + profiles: + - dev + - ingest image: elasticsearch:2.4 volumes: - type: volume @@ -178,6 +190,8 @@ services: # elasticsearch_test: + profiles: + - test image: elasticsearch:2.4 volumes: - type: volume @@ -204,6 +218,9 @@ services: # pghost: + profiles: + - dev + - test hostname: pghost image: "postgres:${PG_VERSION_TAG:-9.6-alpine}" build: From a21ea7017efc466e39db6aa58cd25fcb1d307e85 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 15 Oct 2024 22:51:15 -0400 Subject: [PATCH 32/45] wait for Pg first It should be available before Elasticsearch --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b0bb40f..1aa1e81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,8 +36,8 @@ services: env_file: - .env command: > - /metacpan-api/wait-for-es.sh http://elasticsearch:9200 "" -- /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- + /metacpan-api/wait-for-es.sh http://elasticsearch:9200 "" -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -73,8 +73,8 @@ services: env_file: - localapi_test.env command: > - /metacpan-api/wait-for-es.sh http://elasticsearch_test:9200 "" -- /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- + /metacpan-api/wait-for-es.sh http://elasticsearch_test:9200 "" -- ${API_SERVER} ./bin/api.pl volumes: - type: volume From 1fc094eeaa9befba3d44b65b8eca9e8989f705db Mon Sep 17 00:00:00 2001 From: Cesar Tessarin Date: Fri, 18 Oct 2024 09:11:07 -0300 Subject: [PATCH 33/45] Add missing index file on partial CPAN One of the local operations commonly used for testing the API, performed by `bin/metacpan author`, requires the find-ls.gz file, which was not included in the default partial CPAN. By adding this file to the loading script, testing becomes easier and eliminates the need for manual intervention to include the index. --- bin/partial-cpan-mirror.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/partial-cpan-mirror.sh b/bin/partial-cpan-mirror.sh index d901723..233b344 100755 --- a/bin/partial-cpan-mirror.sh +++ b/bin/partial-cpan-mirror.sh @@ -16,3 +16,4 @@ $RSYNC $PATH/authors/0* $MINICPAN/ $RSYNC $PATH/modules/0* $MINICPAN/ $RSYNC $PATH/indices/mirrors.json $MINICPAN/ +$RSYNC $PATH/indices/find-ls.gz $MINICPAN/ From f215ea5369fb3cbec1bd9a52ceff81de12c87224 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 22 Oct 2024 21:25:38 -0400 Subject: [PATCH 34/45] Revert "specify image name for web-server" --- compose/web.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/web.yml b/compose/web.yml index 1f490a9..72ceb0e 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -1,7 +1,6 @@ --- services: web-server: - image: metacpan/metacpan-web:latest ports: - "5001:80" networks: From 9c8965be6b4f20de324afe68640da241308bbbbc Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Tue, 22 Oct 2024 21:16:28 -0400 Subject: [PATCH 35/45] Add a cloud-es full-stack profile This allows us to test a full stack environment with a cloud-based elasticsearch --- compose/web.yml | 2 ++ docker-compose.yml | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/compose/web.yml b/compose/web.yml index 72ceb0e..f6becf1 100644 --- a/compose/web.yml +++ b/compose/web.yml @@ -1,6 +1,8 @@ --- services: web-server: + profiles: + - cloud-es ports: - "5001:80" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 1aa1e81..648cf1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,41 @@ services: # |_| # + cloud_api: + profiles: + - cloud-es + depends_on: + - pghost + image: metacpan/metacpan-api:latest + build: + context: ./src/metacpan-api + # put variables for compose inside a .env file + # use env_file for variables to be set inside the container + env_file: + - .env + command: > + /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- + ${API_SERVER} ./bin/api.pl + volumes: + - type: volume + source: cpan + target: /CPAN + - type: bind + source: ./src/metacpan-api + target: /metacpan-api + - type: bind + source: ./bin/index-cpan.sh + target: /bin/index-cpan.sh + read_only: true + - type: bind + source: ./bin/partial-cpan-mirror.sh + target: /bin/partial-cpan-mirror.sh + read_only: true + ports: + - "5000:5000" + networks: + - database + - web-network api: profiles: - dev @@ -219,6 +254,7 @@ services: pghost: profiles: + - cloud-es - dev - test hostname: pghost From 1209167b51e9f3139b3b021f012b56ac05c56f58 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 25 Oct 2024 11:17:49 +0200 Subject: [PATCH 36/45] remove no longer used Elasticsearch scripts --- docker-compose.yml | 8 -------- .../scripts/prefer_shorter_module_names_100.groovy | 1 - .../scripts/prefer_shorter_module_names_400.groovy | 2 -- elasticsearch/scripts/score_version_numified.groovy | 1 - elasticsearch/scripts/status_is_latest.groovy | 1 - 5 files changed, 13 deletions(-) delete mode 100644 elasticsearch/scripts/prefer_shorter_module_names_100.groovy delete mode 100644 elasticsearch/scripts/prefer_shorter_module_names_400.groovy delete mode 100644 elasticsearch/scripts/score_version_numified.groovy delete mode 100644 elasticsearch/scripts/status_is_latest.groovy diff --git a/docker-compose.yml b/docker-compose.yml index 1aa1e81..b0ecc4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,10 +167,6 @@ services: source: ./elasticsearch/metacpan.yml target: /usr/share/elasticsearch/config/metacpan.yml read_only: true - - type: bind - source: ./elasticsearch/scripts - target: /usr/share/elasticsearch/config/scripts - read_only: true ports: - "9200" networks: @@ -201,10 +197,6 @@ services: source: ./elasticsearch/test.yml target: /usr/share/elasticsearch/config/test.yml read_only: true - - type: bind - source: ./elasticsearch/scripts - target: /usr/share/elasticsearch/config/scripts - read_only: true ports: - "9200" networks: diff --git a/elasticsearch/scripts/prefer_shorter_module_names_100.groovy b/elasticsearch/scripts/prefer_shorter_module_names_100.groovy deleted file mode 100644 index f99da4d..0000000 --- a/elasticsearch/scripts/prefer_shorter_module_names_100.groovy +++ /dev/null @@ -1 +0,0 @@ -_score - doc.documentation.value.length().toDouble()/100; \ No newline at end of file diff --git a/elasticsearch/scripts/prefer_shorter_module_names_400.groovy b/elasticsearch/scripts/prefer_shorter_module_names_400.groovy deleted file mode 100644 index 04c787b..0000000 --- a/elasticsearch/scripts/prefer_shorter_module_names_400.groovy +++ /dev/null @@ -1,2 +0,0 @@ -len = (doc.documentation.empty ? 26 : doc.documentation.value.length()); -_score - len.toDouble()/400; \ No newline at end of file diff --git a/elasticsearch/scripts/score_version_numified.groovy b/elasticsearch/scripts/score_version_numified.groovy deleted file mode 100644 index 65032ea..0000000 --- a/elasticsearch/scripts/score_version_numified.groovy +++ /dev/null @@ -1 +0,0 @@ -doc.module.version_numified.value; \ No newline at end of file diff --git a/elasticsearch/scripts/status_is_latest.groovy b/elasticsearch/scripts/status_is_latest.groovy deleted file mode 100644 index ac7ba92..0000000 --- a/elasticsearch/scripts/status_is_latest.groovy +++ /dev/null @@ -1 +0,0 @@ -doc.status.value == 'latest'; \ No newline at end of file From e7645d6e5512330362234f38214012dcf0b15448 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 Oct 2024 01:27:10 +0100 Subject: [PATCH 37/45] configure elasticsearch using environment The elasticsearch docker images accept configuration via the environment. This is easier than adding config files to the image, which as far as I can tell don't get used. --- docker-compose.yml | 12 ++++-------- elasticsearch/metacpan.yml | 29 ----------------------------- elasticsearch/test.yml | 29 ----------------------------- 3 files changed, 4 insertions(+), 66 deletions(-) delete mode 100644 elasticsearch/metacpan.yml delete mode 100644 elasticsearch/test.yml diff --git a/docker-compose.yml b/docker-compose.yml index 05ee833..07f43ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,14 +194,12 @@ services: - dev - ingest image: elasticsearch:2.4 + environment: + - discovery.type=single-node volumes: - type: volume source: elasticsearch target: /usr/share/elasticsearch/data - - type: bind - source: ./elasticsearch/metacpan.yml - target: /usr/share/elasticsearch/config/metacpan.yml - read_only: true ports: - "9200" networks: @@ -224,14 +222,12 @@ services: profiles: - test image: elasticsearch:2.4 + environment: + - discovery.type=single-node volumes: - type: volume source: elasticsearch_test target: /usr/share/elasticsearch/data - - type: bind - source: ./elasticsearch/test.yml - target: /usr/share/elasticsearch/config/test.yml - read_only: true ports: - "9200" networks: diff --git a/elasticsearch/metacpan.yml b/elasticsearch/metacpan.yml deleted file mode 100644 index 8eafdd9..0000000 --- a/elasticsearch/metacpan.yml +++ /dev/null @@ -1,29 +0,0 @@ -cluster.name: 'dev' - -index: - number_of_replicas: 0 - number_of_shards: 1 - search: - slowlog: - threshold: - query: - warn: 10s - info: 2s - fetch: - warn: 1s - -gateway: - recover_after_nodes: 1 - recover_after_time: 3m - expected_nodes: 1 - -node: - max_local_storage_nodes: 1 - -discovery: - zen: - minimum_master_nodes: 1 - ping: - multicast.enabled: false - unicast.hosts: - - "elasticsearch:9300" diff --git a/elasticsearch/test.yml b/elasticsearch/test.yml deleted file mode 100644 index 8fa6cd5..0000000 --- a/elasticsearch/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -cluster.name: 'dev-test' - -index: - number_of_replicas: 0 - number_of_shards: 1 - search: - slowlog: - threshold: - query: - warn: 10s - info: 2s - fetch: - warn: 1s - -gateway: - recover_after_nodes: 1 - recover_after_time: 2m - expected_nodes: 1 - -node: - max_local_storage_nodes: 1 - -discovery: - zen: - minimum_master_nodes: 1 - ping: - multicast.enabled: false - unicast.hosts: - - "elasticsearch_test:9300" From 4d030b5fc516c1bd4d7e11996e206e6083f1de05 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 Oct 2024 01:37:10 +0100 Subject: [PATCH 38/45] use health checks rather than custom scripts Rather than having scripts running in the consumer before loading, add a health check, and allow docker compose itself to control the initialization of dependent containers. --- docker-compose.yml | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 07f43ea..42231fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,8 @@ services: profiles: - cloud-es depends_on: - - pghost + pghost: + condition: service_healthy image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api @@ -35,7 +36,6 @@ services: env_file: - .env command: > - /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -57,12 +57,15 @@ services: networks: - database - web-network + api: profiles: - dev depends_on: - - elasticsearch - - pghost + elasticsearch: + condition: service_healthy + pghost: + condition: service_healthy image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api @@ -71,8 +74,6 @@ services: env_file: - .env command: > - /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- - /metacpan-api/wait-for-es.sh http://elasticsearch:9200 "" -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -100,16 +101,16 @@ services: profiles: - test depends_on: - - elasticsearch_test - - pghost + elasticsearch_test: + condition: service_healthy + pghost: + condition: service_healthy image: metacpan/metacpan-api:latest build: context: ./src/metacpan-api env_file: - localapi_test.env command: > - /metacpan-api/wait-for-it.sh -t 15 -s -h ${PG_HOST} -p ${PG_PORT} -- - /metacpan-api/wait-for-es.sh http://elasticsearch_test:9200 "" -- ${API_SERVER} ./bin/api.pl volumes: - type: volume @@ -200,6 +201,10 @@ services: - type: volume source: elasticsearch target: /usr/share/elasticsearch/data + healthcheck: + timeout: 5s + start_period: 60s + test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] ports: - "9200" networks: @@ -228,6 +233,10 @@ services: - type: volume source: elasticsearch_test target: /usr/share/elasticsearch/data + healthcheck: + timeout: 5s + start_period: 60s + test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] ports: - "9200" networks: From e88be061062dd5f6812dba87a1ae08ef721d9ff1 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Wed, 30 Oct 2024 10:42:21 +0100 Subject: [PATCH 39/45] Added ingest test profile Allows running the scripts with elasticsearch_test instance --- docker-compose.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 05ee833..08cad60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,6 +99,7 @@ services: api_test: profiles: - test + - ingest-test depends_on: - elasticsearch_test - pghost @@ -150,6 +151,25 @@ services: networks: - elasticsearch + ingest-test: + profiles: + - ingest-test + image: metacpan/metacpan-ingest:latest + environment: + PLACK_ENV: development + volumes: + - type: volume + source: cpan + target: /CPAN + - type: bind + source: ./configs/metacpan-ingest/metacpan_ingest_local.conf + target: /metacpan-ingest/metacpan_ingest_local.conf + read_only: true + depends_on: + - elasticsearch_test + networks: + - elasticsearch + # __ _ _ __ ___ _ __ # / _` | '__/ _ \ '_ \ # | (_| | | | __/ |_) | @@ -223,6 +243,7 @@ services: elasticsearch_test: profiles: - test + - ingest-test image: elasticsearch:2.4 volumes: - type: volume @@ -249,6 +270,7 @@ services: - cloud-es - dev - test + - ingest-test hostname: pghost image: "postgres:${PG_VERSION_TAG:-9.6-alpine}" build: From 5faa3b690645866f6071ab541dddae921792639a Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Thu, 1 May 2025 12:09:38 +0200 Subject: [PATCH 40/45] Add useful installation tips for a fresh system --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2363ef6..3516dd4 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ of things to be done better. Please use it and create Issues with your problems. ## Quick Start -Install [Docker][0] and [Docker Compose][1] for your platform. [Docker for +Install [Docker][0] and [Docker Compose][1] (v2+) for your platform. [Docker for Mac][2] or [Docker for Windows][3] will install both tools for you, if you are on either of these environments. @@ -60,6 +60,10 @@ on either of these environments. [2]: https://docs.docker.com/docker-for-mac/ [3]: https://docs.docker.com/docker-for-windows/ +On Debian / Ubuntu, install using: + + apt install docker-compose-v2 + On Linux, Docker's default implementation only allows `root` user access to Docker commands and to control containers. In order to allow a regular user to access docker follow the @@ -74,6 +78,10 @@ Docker BuildKit can be enabled by following the [Getting Started](https://docs.docker.com/build/buildkit/#getting-started) instructions. +On Debian / Ubuntu, install using: + + apt install buildx + If you are running a Mac ARM64 system, you will need to manually tell docker to use the x86_64 version of Elasticsearch 2.4. This can be done by running the command: From 3db28bad7de541f4144377c5ad524933c39e5099 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Thu, 1 May 2025 23:39:46 +0200 Subject: [PATCH 41/45] fix README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3516dd4..21d9a25 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ on either of these environments. On Debian / Ubuntu, install using: - apt install docker-compose-v2 + apt install docker-compose-v2 On Linux, Docker's default implementation only allows `root` user access to Docker commands and to control containers. In order to allow a regular user to @@ -80,7 +80,7 @@ instructions. On Debian / Ubuntu, install using: - apt install buildx + apt install docker-buildx If you are running a Mac ARM64 system, you will need to manually tell docker to use the x86_64 version of Elasticsearch 2.4. This can be done by running the From 17f88a7bdf2151d083fc52d5abc6b3574f559bcd Mon Sep 17 00:00:00 2001 From: Erik Huelsmann Date: Fri, 23 May 2025 23:24:41 +0200 Subject: [PATCH 42/45] Fix the order of instructions for initialization Following the steps for initialization, I had to kickstart the web server before I was able to browse through port 5001. Apparently, the web server isn't automatically started. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 21d9a25..7951bd7 100644 --- a/README.md +++ b/README.md @@ -130,15 +130,15 @@ This will prompt you to confirm removing old indices and setting up mappings on the Elasticsearch service (say `YES`). It will then proceed to rsync a partial CPAN in `/CPAN` for its metadata to be imported. -Once the above is done, you should be able to see your local partial CPAN data -in e.g. [http://localhost:5001/recent](http://localhost:5001/recent) and -elsewhere. - -Alternatively, if you just want to hack on the web frontend, you can run this -instead of all the above: +After the initialization above completes, the next step is to start the web +frontend with the following command: docker compose up web-server +Once that is done, you should be able to see your local partial CPAN data +in e.g. [http://localhost:5001/recent](http://localhost:5001/recent) and +elsewhere. + From here, you can proceed and hack on the MetaCPAN code at `src/metacpan-api` and/or `src/metacpan-web` directories, and saving edits will reload the corresponding apps automatically! From 2011e25a40061b314df78d824466607822ceb5f6 Mon Sep 17 00:00:00 2001 From: Erik Huelsmann Date: Sat, 24 May 2025 00:22:07 +0200 Subject: [PATCH 43/45] Remove reference to traefik Since Traefik isn't used... --- README.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/README.md b/README.md index 7951bd7..5f20608 100644 --- a/README.md +++ b/README.md @@ -185,28 +185,6 @@ To access the `psql` command line client in the PostgreSQL container: Each container is responsible for a different service. Some of these services are available in the developer environment via ports on the host system. -We are using [traefik][13] to manage the traffic between services. The current -configuration is: - -- api: [http://api.metacpan.localhost](http://api.metacpan.localhost) -- web: [http://web.metacpan.localhost](http://web.metacpan.localhost) -- grep: [http://grep.metacpan.localhost](http://grep.metacpan.localhost) - -In order to access to the localhost subdomains, you probably have to manually -add these entries in you `/etc/hosts` file. - - # add to /etc/hosts - 127.0.0.1 api.metacpan.localhost - 127.0.0.1 gh.metacpan.localhost - 127.0.0.1 grep.metacpan.localhost - 127.0.0.1 metacpan.localhost - 127.0.0.1 web.metacpan.localhost - -You can access the dashboard configuration via: -[http://metacpan.localhost:8080](http://metacpan.localhost:8080) - -[0]: https://docs.traefik.io/providers/docker/ - #### `web` The local instance of the web front end is accessible via: From 69b77624dba3229ead3f789e85fa8de21b46388a Mon Sep 17 00:00:00 2001 From: Erik Huelsmann Date: Sat, 24 May 2025 00:40:49 +0200 Subject: [PATCH 44/45] Expose grep container ports ... So the port number can be used in the README. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 463cea7..854e883 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -182,6 +182,8 @@ services: profiles: - grep image: metacpan/metacpan-grep-front-end:latest + ports: + - "3000:3000" build: context: ./src/metacpan-grep-front-end volumes: From 6515045774d6f92c172e0f9f4fe898e41906af0d Mon Sep 17 00:00:00 2001 From: Erik Huelsmann Date: Sat, 24 May 2025 00:58:25 +0200 Subject: [PATCH 45/45] Change "grep" access references --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f20608..8bf4b24 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ The PostgreSQL service by default is only accessible from other containers. The grep metacpan front end is accessible via: -- [http://grep.metacpan.localhost](http://grep.metacpan.localhost) +- [http://localhost:3000](http://localhost:3000) Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via [metacpan-cpan-extracted-lite](https://github.com/metacpan/metacpan-cpan-extracted-lite).