diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000..b933d65
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,42 @@
+---
+# Use the latest 2.1 version of CircleCI pipeline process engine. See:
+# https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+# Orchestrate or schedule a set of jobs
+workflows:
+ docker-compose:
+ jobs:
+ - build-and-test
+jobs:
+ build-and-test:
+ machine: true
+ resource_class: large
+ steps:
+ - checkout
+ - run:
+ command: |
+ ./bin/metacpan-docker init
+ name: clone missing repositories
+ - run:
+ command: |
+ docker-compose --verbose --profile test up -d
+ name: compose up
+ # Since we're running docker-compose -d, we don't actually know if
+ # Elasticsearch is available at the time this build step begins. We
+ # probably need to wait for it here, so we'll add our own check.
+ - run:
+ command: |
+ ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test --
+ name: wait for ES
+ - run:
+ name: Run complete MetaCPAN API Test Suite
+ command: |
+ docker-compose --profile test exec -T api_test prove -lr --jobs 4 t
+ docker-compose --profile test down
+ - run:
+ name: Show Docker container logs on Error
+ command: |
+ docker-compose --profile test logs
+ docker stats --no-stream
+ docker ps -a | head
+ when: on_fail
diff --git a/.env b/.env
new file mode 100644
index 0000000..6536994
--- /dev/null
+++ b/.env
@@ -0,0 +1,14 @@
+API_SERVER="morbo -l http://*:5000 -w app.psgi -w bin -w lib -w templates --verbose"
+COLUMNS=80
+ES=elasticsearch:9200
+ES_TEST=elasticsearch_test:9200
+MINICPAN=/CPAN
+MOJO_MODE=development
+NET_ASYNC_HTTP_MAXCONNS=1
+PERL_CARTON_PATH=/carton
+PG_DATABASE=metacpan
+PG_HOST=pghost
+PG_PASSWORD=t00lchain
+PG_PORT=5432
+PG_USER=metacpan
+PLACK_ENV=development
diff --git a/.github/workflows/up.yml b/.github/workflows/up.yml
new file mode 100644
index 0000000..0adfc79
--- /dev/null
+++ b/.github/workflows/up.yml
@@ -0,0 +1,39 @@
+---
+name: docker-compose up
+
+on:
+ push:
+ schedule:
+ # Every Sunday morning
+ - cron: "15 4 * * *"
+jobs:
+ build:
+ name: init and up
+ runs-on: ubuntu-latest
+ strategy:
+ max-parallel: 1
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: init
+ run: bin/metacpan-docker init
+
+ - name: up api
+ run: docker-compose up -d api
+ - name: down
+ run: docker-compose down
+
+ - name: up api_test
+ run: docker-compose up -d api_test
+ - name: down
+ run: docker-compose down
+
+ - name: up web
+ run: docker-compose up -d web
+ - name: down
+ run: docker-compose down
+
+ - name: up grep
+ run: docker-compose up -d grep
+ - name: down
+ run: docker-compose down
diff --git a/.gitignore b/.gitignore
index f0cd683..27e238f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ nytprof.out
*.o
*.bs
src/
+.DS_Store
diff --git a/README.md b/README.md
index eeac151..8bf4b24 100644
--- a/README.md
+++ b/README.md
@@ -1,152 +1,340 @@
-# Running the MetaCPAN stack with Docker (via Docker Compose)
-
-**Notice**: This project is in experimental stage. It works, but there
-are a lot of things to be done better. Please use it and create Issues
-with your problems.
+# MetaCPAN Docker
+
+[](https://circleci.com/gh/metacpan/metacpan-docker)
+
+
+
+
+
+- [Running the MetaCPAN stack with Docker (via Docker Compose)](#running-the-metacpan-stack-with-docker-via-docker-compose)
+- [Quick Start](#quick-start)
+- [Working with Containers](#working-with-containers)
+ - [Building Containers](#building-containers)
+ - [Accessing Containers](#accessing-containers)
+ - [Accessing Services](#accessing-services)
+ - [`web`](#web)
+ - [`api`](#api)
+ - [`Elasticsearch`](#elasticsearch)
+ - [`PostgreSQL`](#postgresql)
+ - [`grep`](#grep)
+- [System architecture](#system-architecture)
+ - [The `bin/metacpan-docker` script](#the-binmetacpan-docker-script)
+ - [`bin/metacpan init`](#binmetacpan-init)
+ - [`bin/metacpan localapi`](#binmetacpan-localapi)
+ - [`bin/metacpan-docker pull`](#binmetacpan-docker-pull)
+ - [`bin/metacpan-docker reset`](#binmetacpan-docker-reset)
+ - [`bin/metacpan-docker` build/up/down/start/stop/run/ps/top...](#binmetacpan-docker-buildupdownstartstoprunpstop)
+ - [Services](#services)
+ - [`web`](#web-1)
+ - [`api`](#api-1)
+ - [`grep`](#grep-1)
+ - [Setting up a partial CPAN in the `api` service](#setting-up-a-partial-cpan-in-the-api-service)
+ - [Bootstrapping the `elasticsearch` indices](#bootstrapping-the-elasticsearch-indices)
+ - [Putting the above all together](#putting-the-above-all-together)
+ - [elasticsearch and elasticsearch_test](#elasticsearch-and-elasticsearch_test)
+- [Tips and tricks](#tips-and-tricks)
+ - [Running your own miniCPAN inside metacpan-docker](#running-your-own-minicpan-inside-metacpan-docker)
+ - [Running tests](#running-tests)
+ - [Updating Carton dependencies](#updating-carton-dependencies)
+ - [Updating the git repositories](#updating-the-git-repositories)
+ - [Running Kibana to peek into Elasticsearch data](#running-kibana-to-peek-into-elasticsearch-data)
+- [Peeking Inside the Container](#peeking-inside-the-container)
+- [To Do](#to-do)
+- [See also](#see-also)
+
+
+
+## Running the MetaCPAN stack with Docker (via Docker Compose)
+
+**Notice**: This project is in experimental stage. It works, but there are a lot
+of things to be done better. Please use it and create Issues with your problems.
## Quick Start
-Install [Docker][0] and [Docker Compose][1] for your
-platform. [Docker for Mac][2] or [Docker for Windows][3] will install
-both tools for you, if you are on either of these environments. It is
-however, recommended to run directly on Linux, for native container
-support, and less issues overall.
+Install [Docker][0] and [Docker Compose][1] (v2+) for your platform. [Docker for
+Mac][2] or [Docker for Windows][3] will install both tools for you, if you are
+on either of these environments.
[0]: https://docs.docker.com/installation
[1]: https://docs.docker.com/compose/install
[2]: https://docs.docker.com/docker-for-mac/
[3]: https://docs.docker.com/docker-for-windows/
-Then, clone this repo and setup the environment:
+On Debian / Ubuntu, install using:
+
+ apt install docker-compose-v2
+
+On Linux, Docker's default implementation only allows `root` user access to
+Docker commands and to control containers. In order to allow a regular user to
+access docker follow the
+[post-installation instructions](https://docs.docker.com/install/linux/linux-postinstall/).
+This document assumes the post-installation steps have been followed for the
+current user.
+
+You will also need Docker buildx, and to enable Docker BuildKit. They should be
+set up by default when using Docker Desktop, but on Linux you may need to
+install them. buildx is the `docker-buildx` package on Debian based systems.
+Docker BuildKit can be enabled by following the
+[Getting Started](https://docs.docker.com/build/buildkit/#getting-started)
+instructions.
+
+On Debian / Ubuntu, install using:
+
+ apt install docker-buildx
+
+If you are running a Mac ARM64 system, you will need to manually tell docker to
+use the x86_64 version of Elasticsearch 2.4. This can be done by running the
+command:
+
+ docker pull elasticsearch:2.4 --platform=linux/x86_64
+
+It is highly recommended that you alias `docker compose` to `fig` (its original
+name) and use it wherever `docker compose` is used. You are going to have to
+type this command a lot.
+
+Then, clone this repo and set up the environment:
git clone https://github.com/metacpan/metacpan-docker.git
cd metacpan-docker
bin/metacpan-docker init
-After this, you can run both the `metacpan-web` frontend on
-http://localhost:5001 and the `metacpan-api` backend on
-http://localhost:5000, with ElasticSearch on http://localhost:9200, via
+The `bin/metacpan-docker init` command clones the source repositories for:
- bin/metacpan-docker localapi up
+- `metacpan-web`
+- `metacpan-api`
+- `metacpan-grep-front-end`
+- `metacpan-cpan-extracted-lite`
-This will build the Docker images for the MetaCPAN and ElasticSearch
-services (which will take a while, especially on a fresh first time
-install of Docker,) and run the services. You'll know when they're
-ready when the services start listening on the ports listed above.
+These repositories are automatically mounted into the appropriate docker
+containers allowing the developer to use their preferred tools to work with the
+source code.
-Don't forget to seed the local `metacpan-api` with a partial CPAN; run
-the following command in a separate terminal to get you up to speed:
+The `docker compose --profile dev up` command on its own will bring up the
+entire stack in the foreground (logs will be displayed).
- bin/metacpan-docker localapi exec api index-cpan.sh
+The `docker compose --profile dev up` command will also fetch the official
+container images from
+[MetaCPAN Docker Hub](https://cloud.docker.com/u/metacpan/repository/list)
+repositories.
+
+This will build the Docker containers for MetaCPAN, PostgreSQL and Elasticsearch
+services (which will take a while, especially on a fresh first time install of
+Docker) and run the services.
+
+Don't forget to seed the local `metacpan-api` with a partial CPAN; run the
+following command in a separate terminal to get yourself up to speed:
+
+ docker compose exec api index-cpan.sh
-This will prompt you to confirm removing old indices and setting up
-mappings on the ElasticSearch service (say `YES`) then proceed to rsync
-a partial CPAN in `/CPAN` for its metadata to be imported.
+This will prompt you to confirm removing old indices and setting up mappings on
+the Elasticsearch service (say `YES`). It will then proceed to rsync a partial
+CPAN in `/CPAN` for its metadata to be imported.
-Once the above is done, you should be able to see your local partial
-CPAN data in e.g. http://localhost:5001/recent and elsewhere.
+After the initialization above completes, the next step is to start the web
+frontend with the following command:
-Alternatively, if you just want to hack on the web frontend, you can run
-this instead of all the above:
+ docker compose up web-server
- docker-compose up
+Once that is done, you should be able to see your local partial CPAN data
+in e.g. [http://localhost:5001/recent](http://localhost:5001/recent) and
+elsewhere.
-From here, you can proceed and hack on the MetaCPAN code at
-`src/metacpan-api` and/or `src/metacpan-web` directories, and saving
-edits will reload the corresponding apps automatically!
+From here, you can proceed and hack on the MetaCPAN code at `src/metacpan-api`
+and/or `src/metacpan-web` directories, and saving edits will reload the
+corresponding apps automatically!
-When done hacking (or, more likely, when you need to rebuild/refresh
-your Docker environment) you can then run
+When done hacking (or, more likely, when you need to rebuild/refresh your Docker
+environment) you can then run
- bin/metacpan-docker localapi down
- # or, if running the metacpan-web service only
- docker-compose down
+ docker compose --profile dev down
-in another terminal to stop all MetaCPAN services and remove the
-containers.
+in another terminal to stop all MetaCPAN services and remove the containers.
For further details, read on!
+## Working with Containers
+
+### Building Containers
+
+You can (re)build arbitrary containers. For instance, if you want to rebuild the
+`api` container:
+
+ docker compose build api
+
+### Accessing Containers
+
+Containers are accessible via the `docker compose exec` command followed by the
+container and then the command to execute. For example, to start a shell prompt
+in the `api` container:
+
+ docker compose exec api /bin/bash
+
+Executing tests via `prove` inside the API container:
+
+ docker compose exec api_test prove -lvr \
+ t/00_setup.t \
+ t/01_darkpan.t \
+ t/api/controller/cover.t
+
+To access the `psql` command line client in the PostgreSQL container:
+
+ docker compose exec pgdb psql
+
+### Accessing Services
+
+Each container is responsible for a different service. Some of these services
+are available in the developer environment via ports on the host system.
+
+#### `web`
+
+The local instance of the web front end is accessible via:
+
+- [http://localhost:5001](http://localhost:5001)
+- [http://web.metacpan.localhost](http://web.metacpan.localhost)
+
+#### `api`
+
+- [http://localhost:5000](http://localhost:5000)
+- [http://api.metacpan.localhost](http://api.metacpan.localhost)
+
+#### `Elasticsearch`
+
+The `elasticsearch` and `elasticsearch_test` containers are not exposed
+directly. They are available via the `api` and `api_test` containers.
+
+You can query the `elasticsearch` container via:
+
+ docker compose exec elasticsearch curl http://localhost:9200
+
+You can query the `elasticsearch_test` container via:
+
+ docker compose exec elasticsearch_test curl http://localhost:9200
+
+#### `PostgreSQL`
+
+The PostgreSQL service by default is only accessible from other containers.
+
+#### `grep`
+
+The grep metacpan front end is accessible via:
+
+- [http://localhost:3000](http://localhost:3000)
+
+Note: this is using a smaller, frozen version of `metacpan-cpan-extracted` via
+[metacpan-cpan-extracted-lite](https://github.com/metacpan/metacpan-cpan-extracted-lite).
+
## System architecture
The system consists of several services that live in docker containers:
- * `web` — the web interface on http://localhost:5001
- * `api` — the main server on http://localhost:5000
- * `elasticsearch` — database on http://localhost:9200
- * `elasticsearch_test` — test database on http://localhost:9300
-
+- `web` — the web interface on [http://localhost:5001](http://localhost:5001)
+- `api` — the main server on [http://localhost:5000](http://localhost:5000)
+- `api_test` — the api server for running tests via `prove`
+- `elasticsearch` — database for `api`
+- `elasticsearch_test` — database for `api_test`
+- `pgdb` - PostgreSQL database container
+- `logspout` - Docker log interface to [honeycomb.io](https://honeycomb.io)
+- `grep` - the web interface for grep.metacpan on
+ [http://localhost:3001](http://localhost:3001)
+
These services use one or more Docker volumes:
- * `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN`
- * `metacpan_elasticsearch`: holds the ElasticSearch database files
- * `metacpan_elasticsearch_test`: holds the ElasticSearch test database
- files
- * `metacpan_api_carton` and `metacpan_web_carton`: holds the
- dependencies installed by [Carton][4] for the `api` and `web`
- services, respectively; mounted on `/carton` instead of `local`, to
- prevent clashing with the host user's Carton
-
+- `metacpan_cpan`: holds the CPAN archive, mounted in `/CPAN`
+- `metacpan_elasticsearch`: holds the Elasticsearch database files
+- `metacpan_elasticsearch_test`: holds the Elasticsearch test database files
+- `metacpan_api_carton` and `metacpan_web_carton`: holds the dependencies
+ installed by [Carton][4] for the `api` and `web` services, respectively;
+ mounted on `/carton` instead of `local`, to prevent clashing with the host
+ user's Carton
+- `metacpan_git_shared`: points to the git repo containing all extracted CPAN
+ versions. This is mounted in `/shared/metacpan_git`. This can be either
+ `metacpan-cpan-extracted` or `metacpan-cpan-extracted-lite`. The volume is
+ bound to the local repo at `${PWD}/src/metacpan-cpan-extracted`.
+
[4]: https://metacpan.org/pod/Carton
-
-Docker Compose is used to, uh, _compose_ them all together into one
-system. Using `docker-compose` directly is a mouthful, however, so
-putting this all together is done via the `bin/metacpan-docker` script
-to simplify setup and usage (and to get you started hacking on the
-MetaCPAN sooner!)
+
+Docker Compose is used to, uh, _compose_ them all together into one system.
+Using `docker compose` directly is a mouthful, however, so putting this all
+together is done via the `bin/metacpan-docker` script to simplify setup and
+usage (and to get you started hacking on the MetaCPAN sooner!)
### The `bin/metacpan-docker` script
-`bin/metacpan-docker` is a thin wrapper for the `docker-compose`
-command, providing the environment variables necessary to run a basic
-MetaCPAN environment. It provides these subcommands:
+`bin/metacpan-docker` is a thin wrapper around the `docker compose` command,
+providing the environment variables necessary to run a basic MetaCPAN
+environment. It provides these subcommands:
#### `bin/metacpan init`
-The `init` subcommand basically clones the [metacpan-api][5]
-and [metacpan-web][6] repositories, and sets up the git commit hooks for
-each of them, in preparation for future `docker-compose` or
+The `init` subcommand basically clones the [metacpan-api][5] and
+[metacpan-web][6] repositories, and sets up the git commit hooks for each of
+them, in preparation for future `docker compose` or
`bin/metacpan-docker localapi` commands.
+It also clones the `metacpan-grep-front-end` and `metacpan-cpan-extracted-lite`
+repositories.
+
[5]: https://github.com/metacpan/metacpan-api
[6]: https://github.com/metacpan/metacpan-web
#### `bin/metacpan localapi`
-The `localapi` subcommand adds the necessary configuration for
-`docker-compose` to run both the `metacpan-web` and `metacpan-api`
-services, along with `elasticsearch` and Docker volumes. Under the
-hood, it customizes the `COMPOSE_FILE` and `COMPOSE_PROJECT_NAME`
-environment variables used by `docker-compose` to use additional YAML
-configuration files aside from the default `docker-compose.yml`.
+The `localapi` subcommand adds the necessary configuration for `docker compose`
+to run both the `metacpan-web` and `metacpan-api` services, along with
+`elasticsearch` and Docker volumes. Under the hood, it customizes the
+`COMPOSE_FILE` and `COMPOSE_PROJECT_NAME` environment variables used by
+`docker compose` to use additional YAML configuration files aside from the
+default `docker compose.yml`.
+
+#### `bin/metacpan-docker pull`
+
+This is used to update all the git repository in `src/*`. This will stay on your
+current local branch.
+
+#### `bin/metacpan-docker reset`
+
+This is used to reset all the git repositories in `src/*` to their latest
+version on `upstream/master`. This will fail if you have some uncommitted local
+changes. You should then commit or cancel your changes before re-running the
+command.
#### `bin/metacpan-docker` build/up/down/start/stop/run/ps/top...
-As noted earlier, `bin/metacpan-docker` is a thin wrapper to
-`docker-compose`, so commands like `up`, `down`, and `run` will work as
-expected from `docker-compose`. See the [docker-compose docs][7] for an
-overview of available commands.
+As noted earlier, `bin/metacpan-docker` is a thin wrapper around
+`docker compose`, so commands like `up`, `down`, and `run` will work as expected
+from `docker compose`. See the [docker compose docs][7] for an overview of
+available commands.
-[7]: https://docs.docker.com/compose/reference/overview/#command-options-overview-and-help
+[7]:
+ https://docs.docker.com/compose/reference/overview/#command-options-overview-and-help
### Services
#### `web`
-The `web` service is a checkout of `metacpan-web`, built as a Docker
-image. Running this service alone is enough if you want to just hack on
-the frontend, since by default the service is configured to talk to
-https://fastapi.metacpan.org for its backend; if this is what you want,
-then you can simply invoke `docker-compose up`.
+The `web` service is a checkout of `metacpan-web`, built as a Docker image.
+Running this service alone is enough if you want to just hack on the frontend,
+since by default the service is configured to talk to
+[https://fastapi.metacpan.org](https://fastapi.metacpan.org) for its backend; if
+this is what you want, then you can simply invoke `docker compose up` or
+`docker compose up web`.
#### `api`
-The `api` service is a checkout of `metacpan-api`, built as a Docker
-image, just like the `web` service.
+The `api` service is a checkout of `metacpan-api`, built as a Docker image, just
+like the `web` service.
If using this service to run a local backend, you will need to run some
-additional commands in a separate terminal once `bin/metacpan-docker
-localapi up` runs:
+additional commands in a separate terminal once
+`bin/metacpan-docker localapi up` runs.
+
+#### `grep`
+
+The `grep` service is a checkout of `metacpan-grep-front-end`, built as a Docker
+image. Note that this is using the `metacpan_git_shared` volume, which requires
+the git repo for `metacpan-cpan-extracted` which can be initialized by running:
+
+ ./bin/metacpan-docker init
##### Setting up a partial CPAN in the `api` service
@@ -154,10 +342,9 @@ Running
bin/metacpan-docker localapi exec api partial-cpan-mirror.sh
-will `rsync` modules selected CPAN authors, plus the package and author
-indices, into the `api` service's `/CPAN` directory. This is nearly
-equivalent to the same script in the [metacpan-developer][8] repository.
-
+will `rsync` modules from selected CPAN authors, plus the package and author
+indices, into the `api` service's `/CPAN` directory. This is nearly equivalent
+to the same script in the (now deprecated) [metacpan-developer][8] repository.
[8]: https://github.com/metacpan/metacpan-developer
##### Bootstrapping the `elasticsearch` indices
@@ -169,23 +356,24 @@ Running
bin/metacpan-docker localapi exec api bin/run bin/metacpan latest
bin/metacpan-docker localapi exec api bin/run bin/metacpan author
-in sequence will create the indices and mappings in the `elasticsearch`
-service, and import the `/CPAN` data into `elasticsearch`.
+in sequence will create the indices and mappings in the `elasticsearch` service,
+and import the `/CPAN` data into `elasticsearch`.
##### Putting the above all together
-If you're impatient or lazy to do all the above, just running
+If you're impatient or too lazy to do all the above, just running
bin/metacpan-docker localapi exec api index-cpan.sh
-
+
instead will set it all up for you.
#### elasticsearch and elasticsearch_test
-The `elasticsearch` and `elasticsearch_test` services uses the
-official [ElasticSearch Docker image][9], configured with settings and
-scripts taken from the [metacpan-puppet][10] repository. It is depended
-on by the `api` service.
+The `elasticsearch` and `elasticsearch_test` services use the official
+[Elasticsearch Docker image][9], configured with settings and scripts taken from
+the [metacpan-puppet][10] repository. The `api` service depends on the
+`elasticsearch` service and the `api_test` service depends on the
+`elasticsearch_test` services.
[9]: https://store.docker.com/images/elasticsearch
[10]: https://github.com/metacpan/metacpan-puppet
@@ -194,10 +382,10 @@ on by the `api` service.
### Running your own miniCPAN inside metacpan-docker
-Suppose you have a local minicpan in `/home/ftp/pub/CPAN`. If you would
-like to use this in metacpan-docker, then edit the
-`docker-compose.localapi.yml` to change the `api` service's volume
-mounts to use your local minicpan as `/CPAN`, e.g.:
+Suppose you have a local minicpan in `/home/ftp/pub/CPAN`. If you would like to
+use this in `metacpan-docker`, then edit the `docker compose.localapi.yml` to
+change the `api` service's volume mounts to use your local minicpan as `/CPAN`,
+e.g.:
```yaml
services:
@@ -207,9 +395,9 @@ services:
...
```
-Note that if you want CPAN author data indexed into ElasticSearch, your
-minicpan should include `authors/00whois.xml`. Full indexing would take
-a better part of a day or two, depending on your hardware.
+Note that if you want CPAN author data indexed into Elasticsearch, your minicpan
+should include `authors/00whois.xml`. Full indexing would take a better part of
+a day or two, depending on your hardware.
### Running tests
@@ -217,62 +405,86 @@ Use `bin/metacpan-docker run` and similar:
# Run tests for metacpan-web against fastapi.metacpan.org
bin/metacpan-docker exec web bin/prove
-
- # Run tests for metacpan-web against local api
+
+ # Run tests for metacpan-web against a local api
bin/metacpan-docker localapi exec web bin/prove
- # Run tests for metacpan-api against local elasticsearch_test
+ # Run tests for metacpan-api against a local elasticsearch_test
bin/metacpan-docker localapi exec api bin/prove
### Updating Carton dependencies
-Because both the `api` and `web` services are running inside
-clean [Perl][11] containers, it is possible to maintain a clean set of
-Carton dependencies independent of your host machine's perl. Just
-update the `cpanfile` of the project, and run
+Because both the `api` and `web` services are running inside clean [Perl][11]
+containers, it is possible to maintain a clean set of Carton dependencies
+independent of your host machine's perl. Just update the `cpanfile` of the
+project, and run
bin/metacpan-docker exec web carton install
# or
- bin/metacpan-docker exec api carton install
-
-Due to the way the Compose services are configured, these commands will
-update the corresponding `cpanfile.snapshot` safely, even if you do _or_
-don't have a `local` directory (internally, the containers' `local`
-directory is placed in `/carton` instead, to prevent interfering with
-the host user's own `local` Carton directory.)
+ bin/metacpan-docker exec api carton install
+
+Due to the way the Compose services are configured, these commands will update
+the corresponding `cpanfile.snapshot` safely, even if you do _or_ don't have a
+`local` directory (internally, the containers' `local` directory is placed in
+`/carton` instead, to prevent interfering with the host user's own `local`
+Carton directory.)
+
+### Updating the git repositories
+
+You can use `bin/metacpan-docker pull` to update all `src/*` directories.
[11]: https://github.com/Perl/docker-perl
-
-### Running Kibana to peek into ElasticSearch data
-By default, the `docker-compose.localapi.yml` configures the
-`elasticsearch` service to listen on the Docker host at
-http://localhost:9200, and is also accessible via the Docker `default`
-network address of http://172.17.0.1:9200; you can inspect it via simple
+### Running Kibana to peek into Elasticsearch data
+
+By default, the `docker compose.localapi.yml` configures the `elasticsearch`
+service to listen on the Docker host at
+[http://localhost:9200](http://localhost:9200), and is also accessible via the
+Docker `default` network address of
+[http://172.17.0.1:9200](http://172.17.0.1:9200); you can inspect it via simple
`curl` or `wget` requests, or use a [Kibana][12] container, e.g.
docker run --rm -p 5601:5601 -e ELASTICSEARCH_URL=http://172.17.0.1:9200 -it kibana:4.6
Running the above will provide a Kibana container at
-http://localhost:5601, which you can configure to have it read the
-`cpan*` index in the `elasticsearch` service.
+[http://localhost:5601](http://localhost:5601), which you can configure to have
+it read the `cpan*` index in the `elasticsearch` service.
-It is also certainly possible to run Kibana as part of the compose
-setup, by configuring e.g. a `kibana` service.
+It is also certainly possible to run Kibana as part of the compose setup, by
+configuring e.g. a `kibana` service.
[12]: https://hub.docker.com/_/kibana/
+## Peeking Inside the Container
+
+If you run `docker ps` you'll see the containers. You might see something like:
+
+```bash
+$ docker ps
+CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
+2efb9c475c83 metacpan-web:latest "carton exec plackup…" 12 hours ago Up 12 hours 0.0.0.0:5001->5001/tcp metacpan_web_1
+8850110e06d8 metacpan-api:latest "/wait-for-it.sh db:…" 12 hours ago Up 12 hours 0.0.0.0:5000->5000/tcp metacpan_api_1
+7686d7ea03c6 postgres:9.6-alpine "docker-entrypoint.s…" 12 hours ago Up 12 hours (healthy) 0.0.0.0:5432->5432/tcp metacpan_pgdb_1
+c7de256d29b2 elasticsearch:2.4 "/docker-entrypoint.…" 5 months ago Up 26 hours 0.0.0.0:9200->9200/tcp, 9300/tcp metacpan_elasticsearch_1
+f1e04fe53598 elasticsearch:2.4 "/docker-entrypoint.…" 5 months ago Up 26 hours 9300/tcp, 0.0.0.0:9900->9200/tcp metacpan_elasticsearch_test_1
+```
+
+You can then use the container name to get shell access. For instance, to log in
+to the API container:
+
+`docker exec -it metacpan_api_1 /bin/bash`
+
## To Do
- * Integrate other MetaCPAN services (e.g. github-meets-cpan)
- * Add more Tips and tricks (as we continue hacking MetaCPAN in Docker)
- * Provide a "near-production" Docker Compose configuration, suitable
- for Docker Swarm, and/or
- * Refactor configuration to be suitable for Kubernetes (Google Cloud)
- deployments
+- Integrate all other MetaCPAN services
+- Add more Tips and tricks (as we continue hacking MetaCPAN in Docker)
+- Provide a "near-production" Docker Compose configuration, suitable for Docker
+ Swarm, and/or
+- Refactor configuration to be suitable for Kubernetes (Google Cloud)
+ deployments
## See also
- * [Docker Compose documentation](https://docs.docker.com/compose/overview)
- * [metacpan-developer][7] and [metacpan-puppet][9] from which much
- information about the architecture is based on
+- [Docker Compose documentation](https://docs.docker.com/compose/overview)
+- [metacpan-developer][7] and [metacpan-puppet][9] from which much information
+ about the architecture is based on
diff --git a/bin/index-cpan.sh b/bin/index-cpan.sh
index 6fab08b..c3d99ea 100755
--- a/bin/index-cpan.sh
+++ b/bin/index-cpan.sh
@@ -7,3 +7,4 @@
./bin/run bin/metacpan release /CPAN/authors/id/
./bin/run bin/metacpan latest
./bin/run bin/metacpan author
+./bin/run bin/metacpan permission
diff --git a/bin/metacpan-docker b/bin/metacpan-docker
index 700b440..139b4b4 100755
--- a/bin/metacpan-docker
+++ b/bin/metacpan-docker
@@ -1,19 +1,21 @@
-#!/bin/bash
-# metacpan-docker: simple wrapper for docker-compose running MetaCPAN
+#!/usr/bin/env bash
+# metacpan-docker: simple wrapper for docker compose running MetaCPAN
set -e
+GitRepos=("metacpan-api" "metacpan-web" "metacpan-grep-front-end" "metacpan-cpan-extracted-lite" "metacpan-ingest")
+
# sanity check
type "docker" > /dev/null
-type "docker-compose" > /dev/null
git_clone_and_setup_hooks() {
local repo=$1
(
cd src
- [ -d "$repo" ] || git clone https://github.com/metacpan/$repo.git
- cd $repo
+ [ -d "$repo" ] || git clone "https://github.com/metacpan/$repo.git"
+ cd "$repo"
[ -e git/hooks/pre-commit ] && chmod +x git/hooks/pre-commit
+ [ -d .git/hooks ] || mkdir .git/hooks
cd .git/hooks
ln -sf ../../git/hooks/pre-commit
)
@@ -23,25 +25,84 @@ git_clone_and_setup_hooks() {
init() {
echo "Initializing metacpan-docker repositories:"
mkdir -p src
- for repo in metacpan-api metacpan-web; do
- git_clone_and_setup_hooks $repo
+ for repo in ${GitRepos[@]}; do
+ git_clone_and_setup_hooks "$repo"
done
+
+ [ -e src/metacpan-cpan-extracted ] || ln -s metacpan-cpan-extracted-lite src/metacpan-cpan-extracted
+
+ docker volume create \
+ --opt type=none \
+ --opt device="$PWD/src/metacpan-cpan-extracted" \
+ --opt o=bind \
+ metacpan_git_shared
+
echo "metacpan-docker ready! Run 'bin/metacpan-docker localapi up' to start."
}
+git_update_repo() {
+ local repo=$1
+ (
+ cd "src/$repo"
+ git fetch origin
+ git pull origin master
+ )
+ echo "Repository $repo updated."
+}
+
+git_reset_repo() {
+ local repo=$1
+ echo "Updating repository $repo"
+ (
+ cd "src/$repo"
+ git fetch origin
+ git checkout master
+ git pull origin master
+ )
+ echo "Repository $repo updated."
+}
+
+update() {
+ echo "Updating metacpan-docker repositories from upstream"
+
+ git fetch origin
+ git pull origin master
+
+ for repo in ${GitRepos[@]}; do
+ git_update_repo "$repo"
+ done
+}
+
+reset_repo() {
+ echo "Resetting metacpan-docker repositories:"
+
+ for repo in ${GitRepos[@]}; do
+ git_reset_repo "$repo"
+ done
+}
+
case "x$1" in
- 'xinit')
- init
- exit
- ;;
- 'xlocalapi')
- export COMPOSE_FILE="docker-compose.yml:docker-compose.localapi.yml"
- shift
- ;;
- *)
- ;;
+'xinit')
+ init
+ exit
+ ;;
+'xreset')
+ reset_repo
+ exit
+ ;;
+'xpull')
+ update
+ exit
+ ;;
+'xlocalapi')
+ shift
+ ;;
+'x')
+ init
+ update
+ exit
+ ;;
+*) ;;
esac
-export COMPOSE_PROJECT_NAME="metacpan"
-
-exec docker-compose "$@"
+exec docker compose "$@"
diff --git a/bin/partial-cpan-mirror.sh b/bin/partial-cpan-mirror.sh
index d901723..233b344 100755
--- a/bin/partial-cpan-mirror.sh
+++ b/bin/partial-cpan-mirror.sh
@@ -16,3 +16,4 @@ $RSYNC $PATH/authors/0* $MINICPAN/
$RSYNC $PATH/modules/0* $MINICPAN/
$RSYNC $PATH/indices/mirrors.json $MINICPAN/
+$RSYNC $PATH/indices/find-ls.gz $MINICPAN/
diff --git a/compose/web.yml b/compose/web.yml
new file mode 100644
index 0000000..f6becf1
--- /dev/null
+++ b/compose/web.yml
@@ -0,0 +1,9 @@
+---
+services:
+ web-server:
+ profiles:
+ - cloud-es
+ ports:
+ - "5001:80"
+ networks:
+ - web-network
diff --git a/configs/metacpan-api/metacpan.pl b/configs/metacpan-api/metacpan.pl
deleted file mode 100644
index 8821e2e..0000000
--- a/configs/metacpan-api/metacpan.pl
+++ /dev/null
@@ -1,23 +0,0 @@
-# do not edit this file
-# create etc/metacpan_local.pl instead
-use FindBin;
-
-{
- # ElasticSearch instance, can be either a single server
- # or an arrayref of servers
- es => 'elasticsearch:9200',
- # the port of the api server
- port => '5000',
- # log level
- level => 'info',
- # appender for Log4perl
- # default layout is "%d %p{1} %c: %m{chomp}%n"
- # can be overridden using the layout key
- # defining logger in metacpan_local.pl will
- # override and not append to this configuration
- logger => [{
- class => 'Log::Log4perl::Appender::File',
- filename => $FindBin::RealBin . '/../var/log/metacpan.log',
- syswrite => 1,
- }]
-}
diff --git a/configs/metacpan-api/metacpan_server.conf b/configs/metacpan-api/metacpan_server.conf
deleted file mode 100644
index c064146..0000000
--- a/configs/metacpan-api/metacpan_server.conf
+++ /dev/null
@@ -1,23 +0,0 @@
-git /usr/bin/git
-cpan /CPAN
-
-
- servers elasticsearch:9200
-
-
-
- servers elasticsearch:9200
-
-
-
- servers elasticsearch:9200
-
-
-
- servers elasticsearch:9200
-
-
-
- # required for server startup -- override this in metacpan_server_local.conf
- private_key 59125ffc09413eed3f2a2c07a37c7a44b95633e2
-
diff --git a/configs/metacpan-ingest/metacpan_ingest_local.conf b/configs/metacpan-ingest/metacpan_ingest_local.conf
new file mode 100644
index 0000000..aed640c
--- /dev/null
+++ b/configs/metacpan-ingest/metacpan_ingest_local.conf
@@ -0,0 +1 @@
+cpan = /CPAN
diff --git a/configs/metacpan-web/metacpan_web.conf b/configs/metacpan-web/metacpan_web.conf
deleted file mode 100644
index c04263b..0000000
--- a/configs/metacpan-web/metacpan_web.conf
+++ /dev/null
@@ -1,40 +0,0 @@
-# rename this file to metacpan::web.yml and put a ':' after 'name' if
-# you want to use YAML like in old versions of Catalyst
-name MetaCPAN::Web
-default_view HTML
-
-api = http://api:5000
-api_external = http://localhost:5000
-api_secure = http://api:5000
-api_external_secure = http://localhost:5000
-source_host = http://localhost:5000
-web_host = http://localhost:5001
-consumer_key = metacpan.dev
-cookie_secret = seekrit
-consumer_secret = ClearAirTurbulence
-
-mark_unauthorized_releases = 0
-
-#site_alert_message = The sky is falling.
-
-
- INCLUDE_PATH root/
- TAG_STYLE asp
- PRE_PROCESS preprocess.html
- WRAPPER wrapper.html
- TEMPLATE_EXTENSION .html
- ENCODING utf8
- AUTO_FILTER html
- STAT_TTL 1
- COMPILE_PERL 0
- COMPILE_DIR var/tmp/templates
-
-
-
- WRAPPER ""
-
-
-
- public_key 6LeH2MsSAAAAANwz3AA73Gw5OjCVjT6I51Ev-ior
-
-
diff --git a/docker-compose.localapi.yml b/docker-compose.localapi.yml
deleted file mode 100644
index fd32f2f..0000000
--- a/docker-compose.localapi.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-version: '3'
-services:
- web:
- depends_on:
- - api
- environment:
- - NET_ASYNC_HTTP_MAXCONNS=1
- volumes:
- - ./configs/metacpan-web/metacpan_web.conf:/metacpan-web/metacpan_web.conf
- api:
- depends_on:
- - elasticsearch
- - elasticsearch_test
- image: metacpan-api:latest
- build:
- context: ./src/metacpan-api
- environment:
- - COLUMNS=80
- - ES=elasticsearch:9200
- - ES_TEST=elasticsearch_test:9200
- - MINICPAN=/CPAN
- - PLACK_ENV=development
- volumes:
- - cpan:/CPAN
- - api_carton:/carton
- - ./src/metacpan-api:/metacpan-api
- - ./configs/metacpan-api/metacpan_server.conf:/metacpan-api/metacpan_server.conf
- - ./configs/metacpan-api/metacpan.pl:/metacpan-api/etc/metacpan.pl
- - ./bin/index-cpan.sh:/bin/index-cpan.sh
- - ./bin/partial-cpan-mirror.sh:/bin/partial-cpan-mirror.sh
- ports:
- - "5000:5000"
- networks:
- - elasticsearch
- - web
- elasticsearch:
- image: elasticsearch:2.4
- volumes:
- - elasticsearch:/usr/share/elasticsearch/data
- - ./elasticsearch/metacpan.yml:/usr/share/elasticsearch/config/metacpan.yml
- - ./elasticsearch/scripts:/usr/share/elasticsearch/config/scripts
- ports:
- - "9200:9200"
- networks:
- - elasticsearch
- elasticsearch_test:
- image: elasticsearch:2.4
- volumes:
- - elasticsearch_test:/usr/share/elasticsearch/data
- - ./elasticsearch/test.yml:/usr/share/elasticsearch/config/test.yml
- - ./elasticsearch/scripts:/usr/share/elasticsearch/config/scripts
- ports:
- - "9900:9200"
- networks:
- - elasticsearch
-
-networks:
- elasticsearch:
-
-volumes:
- api_carton:
- cpan:
- elasticsearch:
- elasticsearch_test:
diff --git a/docker-compose.yml b/docker-compose.yml
index 0abe35c..854e883 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,22 +1,338 @@
-version: '3'
+---
+x-project:
+ name: metacpan
+# ____ _____ ______ _____ ____ _____ ____
+# / ___|| ____| _ \ \ / /_ _/ ___| ____/ ___|
+# \___ \| _| | |_) \ \ / / | | | | _| \___ \
+# ___) | |___| _ < \ V / | | |___| |___ ___) |
+# |____/|_____|_| \_\ \_/ |___\____|_____|____/
+#
+
+include:
+ - path:
+ - src/metacpan-web/docker-compose.yml
+ - compose/web.yml
+
services:
- web:
- image: metacpan-web:latest
+ # _
+ # __ _ _ __ (_)
+ # / _` | '_ \| |
+ # | (_| | |_) | |
+ # \__,_| .__/|_|
+ # |_|
+ #
+
+ cloud_api:
+ profiles:
+ - cloud-es
+ depends_on:
+ pghost:
+ condition: service_healthy
+ image: metacpan/metacpan-api:latest
+ build:
+ context: ./src/metacpan-api
+ # put variables for compose inside a .env file
+ # use env_file for variables to be set inside the container
+ env_file:
+ - .env
+ command: >
+ ${API_SERVER} ./bin/api.pl
+ volumes:
+ - type: volume
+ source: cpan
+ target: /CPAN
+ - type: bind
+ source: ./src/metacpan-api
+ target: /metacpan-api
+ - type: bind
+ source: ./bin/index-cpan.sh
+ target: /bin/index-cpan.sh
+ read_only: true
+ - type: bind
+ source: ./bin/partial-cpan-mirror.sh
+ target: /bin/partial-cpan-mirror.sh
+ read_only: true
+ ports:
+ - "5000:5000"
+ networks:
+ - database
+ - web-network
+
+ api:
+ profiles:
+ - dev
+ depends_on:
+ elasticsearch:
+ condition: service_healthy
+ pghost:
+ condition: service_healthy
+ image: metacpan/metacpan-api:latest
+ build:
+ context: ./src/metacpan-api
+ # put variables for compose inside a .env file
+ # use env_file for variables to be set inside the container
+ env_file:
+ - .env
+ command: >
+ ${API_SERVER} ./bin/api.pl
+ volumes:
+ - type: volume
+ source: cpan
+ target: /CPAN
+ - type: bind
+ source: ./src/metacpan-api
+ target: /metacpan-api
+ - type: bind
+ source: ./bin/index-cpan.sh
+ target: /bin/index-cpan.sh
+ read_only: true
+ - type: bind
+ source: ./bin/partial-cpan-mirror.sh
+ target: /bin/partial-cpan-mirror.sh
+ read_only: true
+ ports:
+ - "5000:5000"
+ networks:
+ - database
+ - elasticsearch
+ - web-network
+
+ api_test:
+ profiles:
+ - test
+ - ingest-test
+ depends_on:
+ elasticsearch_test:
+ condition: service_healthy
+ pghost:
+ condition: service_healthy
+ image: metacpan/metacpan-api:latest
+ build:
+ context: ./src/metacpan-api
+ env_file:
+ - localapi_test.env
+ command: >
+ ${API_SERVER} ./bin/api.pl
+ volumes:
+ - type: volume
+ source: cpan
+ target: /CPAN
+ - type: bind
+ source: ./src/metacpan-api
+ target: /metacpan-api
+ - type: bind
+ source: ./bin/index-cpan.sh
+ target: /bin/index-cpan.sh
+ read_only: true
+ - type: bind
+ source: ./bin/partial-cpan-mirror.sh
+ target: /bin/partial-cpan-mirror.sh
+ read_only: true
+ ports:
+ - "5000"
+ networks:
+ - database
+ - elasticsearch
+
+ ingest:
+ profiles:
+ - dev
+ - ingest
+ image: metacpan/metacpan-ingest:latest
+ volumes:
+ - type: volume
+ source: cpan
+ target: /CPAN
+ - type: bind
+ source: ./configs/metacpan-ingest/metacpan_ingest_local.conf
+ target: /metacpan-ingest/metacpan_ingest_local.conf
+ read_only: true
+ depends_on:
+ - elasticsearch
+ networks:
+ - elasticsearch
+
+ ingest-test:
+ profiles:
+ - ingest-test
+ image: metacpan/metacpan-ingest:latest
+ environment:
+ PLACK_ENV: development
+ volumes:
+ - type: volume
+ source: cpan
+ target: /CPAN
+ - type: bind
+ source: ./configs/metacpan-ingest/metacpan_ingest_local.conf
+ target: /metacpan-ingest/metacpan_ingest_local.conf
+ read_only: true
+ depends_on:
+ - elasticsearch_test
+ networks:
+ - elasticsearch
+
+ # __ _ _ __ ___ _ __
+ # / _` | '__/ _ \ '_ \
+ # | (_| | | | __/ |_) |
+ # \__, |_| \___| .__/
+ # __/ | | |
+ # |___/ |_|
+
+ grep:
+ profiles:
+ - grep
+ image: metacpan/metacpan-grep-front-end:latest
+ ports:
+ - "3000:3000"
build:
- context: ./src/metacpan-web
+ context: ./src/metacpan-grep-front-end
+ volumes:
+ - type: volume
+ source: metacpan_git_shared
+ target: /shared/metacpan_git
+ read_only: true
+ - type: bind
+ source: ./src/metacpan-grep-front-end
+ target: /metacpan-grep-front-end
+ read_only: true
+ env_file:
+ - grep.env
+
+ # ____ _ _____ _ ____ _ ____ _____ ____
+ # | _ \ / \|_ _|/ \ | __ ) / \ / ___|| ____/ ___|
+ # | | | |/ _ \ | | / _ \ | _ \ / _ \ \___ \| _| \___ \
+ # | |_| / ___ \| |/ ___ \| |_) / ___ \ ___) | |___ ___) |
+ # |____/_/ \_\_/_/ \_\____/_/ \_\____/|_____|____/
+ #
+
+ # _ _ _ _
+ # ___| | __ _ ___| |_(_) ___ ___ ___ __ _ _ __ ___| |__
+ # / _ \ |/ _` / __| __| |/ __/ __|/ _ \/ _` | '__/ __| '_ \
+ # | __/ | (_| \__ \ |_| | (__\__ \ __/ (_| | | | (__| | | |
+ # \___|_|\__,_|___/\__|_|\___|___/\___|\__,_|_| \___|_| |_|
+ #
+
+ elasticsearch:
+ profiles:
+ - dev
+ - ingest
+ image: elasticsearch:2.4
+ environment:
+ - discovery.type=single-node
+ volumes:
+ - type: volume
+ source: elasticsearch
+ target: /usr/share/elasticsearch/data
+ healthcheck:
+ timeout: 5s
+ start_period: 60s
+ test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"]
+ ports:
+ - "9200"
+ networks:
+ - elasticsearch
+
+ # _ _ _ _
+ # ___| | __ _ ___| |_(_) ___ ___ ___ __ _ _ __ ___| |__
+ # / _ \ |/ _` / __| __| |/ __/ __|/ _ \/ _` | '__/ __| '_ \
+ # | __/ | (_| \__ \ |_| | (__\__ \ __/ (_| | | | (__| | | |
+ # \___|_|\__,_|___/\__|_|\___|___/\___|\__,_|_| \___|_| |_|
+ #
+ # _ _
+ # | |_ ___ ___| |_
+ # | __/ _ \/ __| __|
+ # | || __/\__ \ |_
+ # \__\___||___/\__|
+ #
+
+ elasticsearch_test:
+ profiles:
+ - test
+ - ingest-test
+ image: elasticsearch:2.4
environment:
- - COLUMNS=80
- - PLACK_ENV=development
+ - discovery.type=single-node
volumes:
- - web_carton:/carton
- - ./src/metacpan-web:/metacpan-web
+ - type: volume
+ source: elasticsearch_test
+ target: /usr/share/elasticsearch/data
+ healthcheck:
+ timeout: 5s
+ start_period: 60s
+ test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"]
ports:
- - "5001:5001"
+ - "9200"
networks:
- - web
+ - elasticsearch
+ # _ _
+ # _ __ ___ ___| |_ __ _ _ __ ___ ___ __ _| |
+ # | '_ \ / _ \/ __| __/ _` | '__/ _ \/ __|/ _` | |
+ # | |_) | (_) \__ \ || (_| | | | __/\__ \ (_| | |
+ # | .__/ \___/|___/\__\__, |_| \___||___/\__, |_|
+ # |_| |___/ |_|
+ #
+
+ pghost:
+ profiles:
+ - cloud-es
+ - dev
+ - test
+ - ingest-test
+ hostname: pghost
+ image: "postgres:${PG_VERSION_TAG:-9.6-alpine}"
+ build:
+ context: "./pg"
+ args:
+ PG_TAG: "${PG_VERSION_TAG:-9.6-alpine}"
+ environment:
+ POSTGRES_PASSWORD: metacpan
+ POSTGRES_USER: metacpan123
+ POSTGRES_DB: metacpan
+ networks:
+ - database
+ healthcheck:
+ interval: 10s
+ timeout: 1s
+ retries: 0
+ start_period: 480s
+ test: ["CMD", "/healthcheck.sh"]
+ volumes:
+ - type: volume
+ source: pghost-data
+ target: /var/lib/postgresql/data
+ - type: bind
+ source: ./pg/docker-entrypoint-initdb.d
+ target: /docker-entrypoint-initdb.d
+ read_only: true
+ - type: bind
+ source: ./pg/healthcheck.sh
+ target: /healthcheck.sh
+ read_only: true
+
+# _ _ _____ _______ _____ ____ _ ______
+# | \ | | ____|_ _\ \ / / _ \| _ \| |/ / ___|
+# | \| | _| | | \ \ /\ / / | | | |_) | ' /\___ \
+# | |\ | |___ | | \ V V /| |_| | _ <| . \ ___) |
+# |_| \_|_____| |_| \_/\_/ \___/|_| \_\_|\_\____/
+#
networks:
- web:
+ database:
+ elasticsearch:
+ web-network:
+
+# __ _____ _ _ _ __ __ _____ ____
+# \ \ / / _ \| | | | | | \/ | ____/ ___|
+# \ \ / / | | | | | | | | |\/| | _| \___ \
+# \ V /| |_| | |__| |_| | | | | |___ ___) |
+# \_/ \___/|_____\___/|_| |_|_____|____/
+#
volumes:
- web_carton:
+ api_carton:
+ cpan:
+ elasticsearch:
+ elasticsearch_test:
+ pghost-data:
+ metacpan_git_shared:
+ external: true
diff --git a/elasticsearch/metacpan.yml b/elasticsearch/metacpan.yml
deleted file mode 100644
index 8eafdd9..0000000
--- a/elasticsearch/metacpan.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-cluster.name: 'dev'
-
-index:
- number_of_replicas: 0
- number_of_shards: 1
- search:
- slowlog:
- threshold:
- query:
- warn: 10s
- info: 2s
- fetch:
- warn: 1s
-
-gateway:
- recover_after_nodes: 1
- recover_after_time: 3m
- expected_nodes: 1
-
-node:
- max_local_storage_nodes: 1
-
-discovery:
- zen:
- minimum_master_nodes: 1
- ping:
- multicast.enabled: false
- unicast.hosts:
- - "elasticsearch:9300"
diff --git a/elasticsearch/scripts/prefer_shorter_module_names_100.groovy b/elasticsearch/scripts/prefer_shorter_module_names_100.groovy
deleted file mode 100644
index f99da4d..0000000
--- a/elasticsearch/scripts/prefer_shorter_module_names_100.groovy
+++ /dev/null
@@ -1 +0,0 @@
-_score - doc.documentation.value.length().toDouble()/100;
\ No newline at end of file
diff --git a/elasticsearch/scripts/prefer_shorter_module_names_400.groovy b/elasticsearch/scripts/prefer_shorter_module_names_400.groovy
deleted file mode 100644
index 04c787b..0000000
--- a/elasticsearch/scripts/prefer_shorter_module_names_400.groovy
+++ /dev/null
@@ -1,2 +0,0 @@
-len = (doc.documentation.empty ? 26 : doc.documentation.value.length());
-_score - len.toDouble()/400;
\ No newline at end of file
diff --git a/elasticsearch/scripts/score_version_numified.groovy b/elasticsearch/scripts/score_version_numified.groovy
deleted file mode 100644
index 65032ea..0000000
--- a/elasticsearch/scripts/score_version_numified.groovy
+++ /dev/null
@@ -1 +0,0 @@
-doc.module.version_numified.value;
\ No newline at end of file
diff --git a/elasticsearch/scripts/status_is_latest.groovy b/elasticsearch/scripts/status_is_latest.groovy
deleted file mode 100644
index ac7ba92..0000000
--- a/elasticsearch/scripts/status_is_latest.groovy
+++ /dev/null
@@ -1 +0,0 @@
-doc.status.value == 'latest';
\ No newline at end of file
diff --git a/elasticsearch/test.yml b/elasticsearch/test.yml
deleted file mode 100644
index 8fa6cd5..0000000
--- a/elasticsearch/test.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-cluster.name: 'dev-test'
-
-index:
- number_of_replicas: 0
- number_of_shards: 1
- search:
- slowlog:
- threshold:
- query:
- warn: 10s
- info: 2s
- fetch:
- warn: 1s
-
-gateway:
- recover_after_nodes: 1
- recover_after_time: 2m
- expected_nodes: 1
-
-node:
- max_local_storage_nodes: 1
-
-discovery:
- zen:
- minimum_master_nodes: 1
- ping:
- multicast.enabled: false
- unicast.hosts:
- - "elasticsearch_test:9300"
diff --git a/github-meets-cpan/environment.json b/github-meets-cpan/environment.json
new file mode 100644
index 0000000..e69de29
diff --git a/grep.env b/grep.env
new file mode 100644
index 0000000..8e1d6a5
--- /dev/null
+++ b/grep.env
@@ -0,0 +1,4 @@
+# grep service - development environment
+
+GREP_SITE_PORT=3001
+GREP_PLACKUP_SERVER_ARGS="-E development -R lib,bin"
diff --git a/localapi_test.env b/localapi_test.env
new file mode 100644
index 0000000..e899080
--- /dev/null
+++ b/localapi_test.env
@@ -0,0 +1,10 @@
+NET_ASYNC_HTTP_MAXCONNS=1
+COLUMNS=80
+ES=elasticsearch_test:9200
+ES_TEST=elasticsearch_test:9200
+HARNESS_ACTIVE=1
+# Instantiate Catalyst models using metacpan_server_testing.conf
+METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing
+MINICPAN=/CPAN
+PERL_CARTON_PATH=/carton
+MOJO_MODE=testing
diff --git a/logging.env b/logging.env
new file mode 100644
index 0000000..0958881
--- /dev/null
+++ b/logging.env
@@ -0,0 +1,8 @@
+## The API KEY
+HONEYCOMB_WRITE_KEY=XXX
+
+## HONEYCOMB_DATASET: 'dev' or 'production'
+HONEYCOMB_DATASET=YYY
+
+## This is honeycomb magic
+ROUTE_URIS=honeycomb://localhost
\ No newline at end of file
diff --git a/pg/Dockerfile b/pg/Dockerfile
new file mode 100644
index 0000000..a087e76
--- /dev/null
+++ b/pg/Dockerfile
@@ -0,0 +1,5 @@
+ARG PG_TAG=9.6-alpine
+
+FROM postgres:${PG_TAG}
+
+VOLUME /logs
diff --git a/pg/docker-entrypoint-initdb.d/100-roles.sql b/pg/docker-entrypoint-initdb.d/100-roles.sql
new file mode 100644
index 0000000..867939c
--- /dev/null
+++ b/pg/docker-entrypoint-initdb.d/100-roles.sql
@@ -0,0 +1,5 @@
+CREATE ROLE metacpan WITH LOGIN PASSWORD 't00lchain';
+CREATE ROLE "metacpan-api" WITH LOGIN;
+
+-- make things easier for when we're poking around from inside the container
+CREATE USER root;
diff --git a/pg/docker-entrypoint-initdb.d/200-minion.sql b/pg/docker-entrypoint-initdb.d/200-minion.sql
new file mode 100644
index 0000000..599486f
--- /dev/null
+++ b/pg/docker-entrypoint-initdb.d/200-minion.sql
@@ -0,0 +1 @@
+CREATE DATABASE minion_queue OWNER metacpan
diff --git a/pg/healthcheck.sh b/pg/healthcheck.sh
new file mode 100755
index 0000000..cde0ff5
--- /dev/null
+++ b/pg/healthcheck.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -eo pipefail
+
+host="$(hostname || echo '127.0.0.1')"
+user="${POSTGRES_USER:-postgres}"
+db="${POSTGRES_DB:-$POSTGRES_USER}"
+export PGPASSWORD="${POSTGRES_PASSWORD:-}"
+
+args=(
+ # force postgres to not use the local unix socket (test "external" connectibility)
+ --host "$host"
+ --username "$user"
+ --dbname "$db"
+ --quiet --no-align --tuples-only
+)
+
+if select="$(echo 'SELECT 1' | psql "${args[@]}")" && [ "$select" = '1' ]; then
+ exit 0
+fi
+
+exit 1