Dockerization updates

Author: jmgasper

.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+.git
+.gitignore

.gitignore

@@ -5,3 +5,4 @@
 
 *.cer
 *.key
+.env

Dockerfile

@@ -0,0 +1,15 @@
+# Use the base image with Node.js 8.12
+FROM node:8.12
+
+# Set working directory for future use
+WORKDIR /topcoder-x-processor
+
+COPY package.json .
+COPY package-lock.json .
+
+# Install the dependencies from package.json
+RUN npm install
+
+COPY . .
+
+ENTRYPOINT npm start

build.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+set -eo pipefail
+
+# This script expects 2 arguments:
+#   $BUILD_TYPE: The build type, like "local", "dev", "test", "prod". Required.
+#   $ENV_FOLDER: The folder containing env files. Optional.
+#
+# If $ENV_FOLDER is not present, then environment values will be used;
+# otherwise the topcoder-x-processor.env.default and topcoder-x-processor.env.$BUILD_TYPE
+# files within $ENV_FOLDER will be used.
+
+BUILD_TYPE=$1
+ENV_FOLDER="$2"
+APP="topcoder-x-processor"
+
+if [ -z "$BUILD_TYPE" ]; then
+  echo "Must provide the build type to build"
+  exit 1
+fi
+
+ENV_FILE=".env.$BUILD_TYPE.tmp"
+DEFAULT_FILE=".env.default.tmp"
+rm -f "$ENV_FILE"
+rm -f "$DEFAULT_FILE"
+
+if [ -z "$ENV_FOLDER" ]; then
+  # Use environment values
+  touch "$ENV_FILE"
+  touch "$DEFAULT_FILE"
+
+  # Loop the environment values, find the ones start with DEFAULT_ and $BUILD_TYPE_
+  # Assuming given $BUILD_TYPE is "prod" and environment has following values:
+  #   DEFAULT_VERSION=1.0
+  #   DEFAULT_SECRET=111111
+  #   PROD_PORT=80
+  #   PROD_SECRET=654321
+  #   LOCAL_PORT=8080
+  #   LOCAL_SECRET=123456
+  # Then the generated .env file will have following content:
+  #   VERSION=1.0
+  #   PORT=80
+  #   SECRET=654321
+  BUILD_TYPE_UPPER=`echo $BUILD_TYPE | awk '{print toupper($0)}'`
+  for var in $(compgen -e); do
+    if [[ "$var" =~ ^DEFAULT\_.* ]]; then
+      tripVar=${var#DEFAULT_}
+      echo $tripVar=${!var} >> "$DEFAULT_FILE"
+    fi
+    if [[ "$var" =~ ^$BUILD_TYPE_UPPER\_.* ]]; then
+      tripVar=${var#"$BUILD_TYPE_UPPER"_}
+      echo $tripVar=${!var} >> "$ENV_FILE"
+    fi
+  done
+
+else
+  if [ ! -d "$ENV_FOLDER" ]; then
+    echo "$ENV_FOLDER does not exist"
+    exit 1
+  fi
+
+  P_DEFAULT_FILE="$ENV_FOLDER/$APP.env.default"
+  P_ENV_FILE="$ENV_FOLDER/$APP.env.$BUILD_TYPE"
+
+  if [ ! -f "$P_DEFAULT_FILE" ]; then
+    echo "$P_DEFAULT_FILE does not exist"
+    exit 1
+  fi
+  cp -f "$P_DEFAULT_FILE" "$DEFAULT_FILE"
+
+  if [ ! -f "$P_ENV_FILE" ]; then
+      echo "Will only use default env since $P_ENV_FILE does not exist"
+      touch "$ENV_FILE"
+  else
+      cp -f "$P_ENV_FILE" "$ENV_FILE"
+  fi
+fi
+
+awk -F= '!a[$1]++' "$ENV_FILE" "$DEFAULT_FILE" > .env
+
+rm -f "$ENV_FILE"
+rm -f "$DEFAULT_FILE"
+
+# Build docker image
+docker build -t $APP:$BUILD_TYPE -f Dockerfile .

config/default.js

@@ -2,6 +2,7 @@
  * Copyright (c) 2017 TopCoder, Inc. All rights reserved.
  */
 'use strict';
+require('dotenv').config();
 const fs = require('fs');
 /**
  * This module is the configuration of the app.
@@ -14,14 +15,14 @@ const fs = require('fs');
 
 module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
-  PARTITION: process.env.PARTITION || 0,
+  PARTITION: process.env.PARTITION ? parseInt(process.env.PARTITION) : 0,
   TOPIC: process.env.TOPIC || 'tc-x-events',
   KAFKA_OPTIONS: {
     connectionString: process.env.KAFKA_HOST || 'localhost:9092',
     ssl: {
       cert: process.env.KAFKA_CLIENT_CERT || fs.readFileSync('./kafka_client.cer'), // eslint-disable-line no-sync
       key: process.env.KAFKA_CLIENT_CERT_KEY || fs.readFileSync('./kafka_client.key'), // eslint-disable-line no-sync
-      passphrase: 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it.
+      passphrase: process.env.KAFKA_CLIENT_CERT_KEY_PASS || 'secret', // NOTE:* This configuration specifies the private key passphrase used while creating it.
     }
   },
   MONGODB_URL: process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/topcoderx',
@@ -39,7 +40,7 @@ module.exports = {
     device: 'Browser'
   },
   TC_AUTHZ_URL: process.env.TC_AUTHZ_URL || 'https://api.topcoder-dev.com/v3/authorizations',
-  NEW_CHALLENGE_TEMPLATE: process.env.NEW_CHALLENGE_TEMPLATE || {
+  NEW_CHALLENGE_TEMPLATE: process.env.NEW_CHALLENGE_TEMPLATE ? JSON.parse(process.env.NEW_CHALLENGE_TEMPLATE) : {
     milestoneId: 1,
     subTrack: 'FIRST_2_FINISH',
     reviewType: 'COMMUNITY',
@@ -66,11 +67,11 @@ module.exports = {
 
   // NOTE: if subTrack is FIRST_2_FINISH,
   // this config has no effect since the ***EndsAt will be set automatically by TC APIs
-  NEW_CHALLENGE_DURATION_IN_DAYS: process.env.NEW_CHALLENGE_DURATION_IN_DAYS || 5,
+  NEW_CHALLENGE_DURATION_IN_DAYS: process.env.NEW_CHALLENGE_DURATION_IN_DAYS ? parseInt(process.env.NEW_CHALLENGE_DURATION_IN_DAYS) : 5,
   // node mailer option
   NODE_MAILER_OPTIONS: {
     host: process.env.SMTP_HOST || process.env.MAILGUN_SMTP_SERVER || 'smtp.gmail.com',
-    port: process.env.SMTP_PORT || process.env.MAILGUN_SMTP_POR || 465,
+    port: process.env.SMTP_PORT || process.env.MAILGUN_SMTP_PORT || 465,
     secure: process.env.SMTP_IS_SECURE || true,
     auth: {
       user: process.env.SMTP_USERNAME || process.env.MAILGUN_SMTP_LOGIN || '',
@@ -84,9 +85,9 @@ module.exports = {
   PAID_ISSUE_LABEL: process.env.PAID_ISSUE_LABEL || 'tcx_Paid',
   FIX_ACCEPTED_ISSUE_LABEL: process.env.FIX_ACCEPTED_ISSUE_LABEL || 'tcx_FixAccepted',
   READY_FOR_REVIEW_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_ReadyForReview',
-  ASSIGNED_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_Assigned',
-  OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.READY_FOR_REVIEW_ISSUE_LABEL || 'tcx_OpenForPickup',
+  ASSIGNED_ISSUE_LABEL: process.env.ASSIGNED_ISSUE_LABEL || 'tcx_Assigned',
+  OPEN_FOR_PICKUP_ISSUE_LABEL: process.env.OPEN_FOR_PICKUP_ISSUE_LABEL || 'tcx_OpenForPickup',
   TC_OR_DETAIL_LINK: process.env.TC_OR_DETAIL_LINK || 'https://software.topcoder-dev.com/review/actions/ViewProjectDetails?pid=',
-  RETRY_COUNT: process.env.RETRY_COUNT || 3,
-  RETRY_INTERVAL: process.env.RETRY_INTERVAL || 120000, // 2 minutes
+  RETRY_COUNT: process.env.RETRY_COUNT ? parseInt(process.env.RETRY_COUNT) : 3,
+  RETRY_INTERVAL: process.env.RETRY_INTERVAL ? parseInt(process.env.RETRY_INTERVAL) : 120000, // 2 minutes
 };

configuration.md

@@ -7,30 +7,31 @@ The following config parameters are supported, they are defined in `config/defau
 | LOG_LEVEL                      | the log level                              |  debug                           |
 | PARTITION                  | The Kafka partition            |  0|
 | MONGODB_URI                            | The MongoDB URI.  This needs to be the same MongoDB used by topcoder-x-receiver, topcoder-x-processor, and topcoder-x-site                           | mongodb://127.0.0.1:27017/topcoderx |
-|TOPIC  | The Kafka topic where events are published.  This must be the same as the configured value for topcoder-x-processor| |
+|TOPIC  | The Kafka topic where events are published.  This must be the same as the configured value for topcoder-x-processor| tc-x-events |
 |KAFKA_OPTIONS | Kafka connection options| |
 |KAFKA_HOST | The Kafka host to connect to| localhost:9092 |
 |KAFKA_CLIENT_CERT | The Kafka SSL certificate to use when connecting| Read from kafka_client.cer file, but this can be set as a string like it is on Heroku |
 |KAFKA_CLIENT_CERT_KEY | The Kafka SSL certificate key to use when connecting| Read from kafka_client.key file, but this can be set as a string like it is on Heroku|
-|TC_DEV_ENV| the flag whether to use topcoder development api or production| false|
+|KAFKA_CLIENT_CERT_KEY_PASS | The passphrase of certificate key | |
+|TC_DEV_ENV| the flag whether to use topcoder development api or production. Set env variable `NODE_ENV` to "production" will cause TC_DEV_ENV evaluated to false. | false|
 | TC_AUTHN_URL | the Topcoder authentication url | https://topcoder-dev.auth0.com/oauth/ro |
 | TC_AUTHN_REQUEST_BODY | the Topcoder authentication request body. This makes use of some environment variables: `TC_USERNAME`, `TC_PASSWORD`, `TC_CLIENT_ID`, `CLIENT_V2CONNECTION` | see `default.js` |
 | TC_AUTHZ_URL | the Topcoder authorization url | https://api.topcoder-dev.com/v3/authorizations |
-| NEW_CHALLENGE_TEMPLATE | the body template for new challenge request. You can change the subTrack, reviewTypes, technologies, .. here | see `default.js` |
+| NEW_CHALLENGE_TEMPLATE | the body template for new challenge request. You can change the subTrack, reviewTypes, technologies, .. here | see `default.js`. Note when setup by environment, it must be a valid JSON string format. |
 | NEW_CHALLENGE_DURATION_IN_DAYS | the duration of new challenge | 5 |
-| NODE_MAILER_OPTIONS| the node mailer smtp options, see [here](https://nodemailer.com/smtp/ for more detail)| see `default.js` |
+| NODE_MAILER_OPTIONS| the node mailer smtp options, see [here](https://nodemailer.com/smtp/ for more detail).This makes use of some environment variables: `SMTP_HOST`/`MAILGUN_SMTP_SERVER`, `SMTP_PORT`/`MAILGUN_SMTP_PORT`, `SMTP_USERNAME`/`MAILGUN_SMTP_LOGIN`,`SMTP_PASSWORD`/`MAILGUN_SMTP_PASSWORD`, `SMTP_IS_SECURE` | see `default.js` |
 |EMAIL_SENDER_ADDRESS| the email sender email address||
 |ISSUE_BID_EMAIL_RECEIVER| the email receiver about bid email||
 |TC_URL| the base URL of topcoder to get the challenge URL| defaults to `https://www.topcoder-dev.com`|
 |GITLAB_API_BASE_URL| the URL for gitlab host| defaults to `https://gitlab.com`|
 |PAID_ISSUE_LABEL|the label name for paid, should be one of the label configured in topcoder x ui|'tcx_Paid'|
 |FIX_ACCEPTED_ISSUE_LABEL|the label name for fix accepted, should be one of the label configured in topcoder x ui|'tcx_FixAccepted'|
+|READY_FOR_REVIEW_ISSUE_LABEL|the label name for ready for review, should be one of the label configured in topcoder x ui|'tcx_ReadyForReview'|
 |ASSIGNED_ISSUE_LABEL| the label name for assigned, should be one of the label configured in topcoder x ui| 'tcx_Assigned'|
 |OPEN_FOR_PICKUP_ISSUE_LABEL| the label name for open for pickup, should be one of the label configured in topcoder x ui| 'tcx_OpenForPickup'|
 |TC_OR_DETAIL_LINK|the link to online review detail of challenge| see `default.js`, OR link for dev environment|
 |RETRY_COUNT| the number of times an event should be retried to process| 3|
 |RETRY_INTERVAL| the interval at which the event should be retried to process in milliseconds | 120000|
-|READY_FOR_REVIEW_ISSUE_LABEL| the label name for ready for review, should be one of the label configured in topcoder x ui|'tcx_ReadyForReview'|
 
 KAFKA_OPTIONS should be object as described in https://github.com/oleksiyk/kafka#ssl
 For using with SSL, the options should be as
@@ -72,7 +73,7 @@ Configure a Github project with a webhook with a format like this: https://<rece
 
 You can test other events, but just validating that an issue.created event is generated in Kafka is enough to smoke test the receiver is set up properly.  
 
-## Github Verification
+## Gitlab Verification
 
 #### Webhook configuration
 

package-lock.json

@@ -26,6 +26,7 @@
   "dependencies": {
     "axios": "^0.16.2",
     "config": "^1.30.0",
+    "dotenv": "^6.0.0",
     "get-parameter-names": "^0.3.0",
     "github": "^12.1.0",
     "global-request-logger": "^0.1.1",