MS v1.3, Auth0 Proxy and other Kafka changes

.circleci/config.yml

@@ -5,10 +5,17 @@ defaults: &defaults
 install_dependency: &install_dependency
           name: Installation of build and deployment dependencies.
           command: |
-            apk add --no-cache bash
+            apk add --no-cache bash curl
             apk add --no-cache jq py-pip sudo
             sudo pip install awscli --upgrade
             sudo pip install docker-compose
+install_deploysuite: &install_deploysuite
+          name: Installation of install_deploysuite.
+          command: |
+            git clone --branch v1.3 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+            cp ./../buildscript/master_deploy.sh .
+            cp ./../buildscript/buildenv.sh .
+            cp ./../buildscript/awsconfiguration.sh .
 restore_cache_settings_for_build: &restore_cache_settings_for_build
   key: docker-node-modules-lp-{{ checksum "package-lock.json" }}
 
@@ -25,15 +32,21 @@ jobs:
       - checkout
       - setup_remote_docker
       - run: *install_dependency
+      - run: *install_deploysuite
       - restore_cache: *restore_cache_settings_for_build
       - run: ./build.sh
       - save_cache: *save_cache_settings
       - deploy:
           name: Running MasterScript.
           command: |
-            git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
-            cp ./../buildscript/master_deploy.sh .
-            ./master_deploy.sh -d ECS -e DEV -t latest -s leaderboard-processor -i leaderboard-processor
+            #git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+            #cp ./../buildscript/master_deploy.sh .
+            #./master_deploy.sh -d ECS -e DEV -t latest -s leaderboard-processor -i leaderboard-processor
+            ./awsconfiguration.sh DEV
+            source awsenvconf
+            ./buildenv.sh -e DEV -b dev-leaderboard-processor-deployvar
+            source buildenvvar
+            ./master_deploy.sh -d ECS -e DEV -t latest -s dev-global-appvar,dev-leaderboard-processor-appvar -i leaderboard-processor
 
   # Build & Deploy against production backend
   "build-prod":
@@ -42,28 +55,35 @@ jobs:
       - checkout
       - setup_remote_docker
       - run: *install_dependency
+      - run: *install_deploysuite      
       - restore_cache: *restore_cache_settings_for_build
       - run: ./build.sh
       - save_cache: *save_cache_settings
       - deploy:
           name: Running MasterScript.
           command: |
-            git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
-            cp ./../buildscript/master_deploy.sh .
-            ./master_deploy.sh -d ECS -e PROD  -t latest -s leaderboard-processor -i leaderboard-processor
+            #git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript
+            #cp ./../buildscript/master_deploy.sh .
+            #./master_deploy.sh -d ECS -e PROD  -t latest -s leaderboard-processor -i leaderboard-processor
+            ./awsconfiguration.sh PROD
+            source awsenvconf
+            ./buildenv.sh -e PROD -b prod-leaderboard-processor-deployvar
+            source buildenvvar
+            ./master_deploy.sh -d ECS -e PROD -t latest -s prod-global-appvar,prod-leaderboard-processor-appvar -i leaderboard-processor
 
 workflows:
   version: 2
   build:
     jobs:
       # Development builds are executed on "develop" branch only.
       - "build-dev":
+          context : org-global 
           filters:
             branches:
               only:
-                - circleci
                 - develop
       - "build-prod":
+          context : org-global 
           filters:
             branches:
               only:

README.md

@@ -13,7 +13,6 @@ The following parameters can be set in config files or in env variables:
 
 - LOG_LEVEL: the log level
 - PORT: the server port
-- KAFKA_GROUP_ID: group id of the consumer; default value: 'leaderboard-processor-group'
 - KAFKA_URL: comma separated Kafka hosts
 - KAFKA_CLIENT_CERT: Kafka connection certificate, optional;
     if not provided, then SSL connection is not used, direct insecure connection is used;

config/default.js

@@ -6,7 +6,6 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
   PORT: process.env.PORT || 3000,
 
-  KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'leaderboard-processor-group',
   KAFKA_URL: process.env.KAFKA_URL || 'localhost:9092',
   // below two params are used for secure Kafka connection, they are optional
   // for the local Kafka, they are not needed
@@ -30,5 +29,6 @@ module.exports = {
   TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME,
   AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
   AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+  AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
   MONGODB_URL: process.env.MONGODB_URL || 'mongodb://localhost:27017/leaderboardDB'
 }

package.json

@@ -27,9 +27,9 @@
     "joi": "^14.0.0",
     "lodash": "^4.17.11",
     "mongoose": "^5.3.3",
-    "no-kafka": "^3.4.3",
+    "no-kafka": "^3.2.4",
     "superagent": "^3.8.3",
-    "tc-core-library-js": "appirio-tech/tc-core-library-js.git#feature/m2mtoken",
+    "tc-core-library-js": "appirio-tech/tc-core-library-js.git#v2.6",
     "topcoder-healthcheck-dropin": "^1.0.3",
     "winston": "^3.1.0"
   },

src/app.js

@@ -4,6 +4,7 @@
 
 require('./bootstrap')
 const config = require('config')
+const _ = require('lodash')
 const logger = require('./common/logger')
 const Kafka = require('no-kafka')
 const healthcheck = require('topcoder-healthcheck-dropin')
@@ -12,14 +13,14 @@ const ProcessorService = require('./services/ProcessorService')
 // start Kafka consumer
 logger.info('Start Kafka consumer.')
 // create consumer
-const options = { connectionString: config.KAFKA_URL, groupId: config.KAFKA_GROUP_ID }
+const options = { connectionString: config.KAFKA_URL }
 if (config.KAFKA_CLIENT_CERT && config.KAFKA_CLIENT_CERT_KEY) {
   options.ssl = { cert: config.KAFKA_CLIENT_CERT, key: config.KAFKA_CLIENT_CERT_KEY }
 }
-const consumer = new Kafka.GroupConsumer(options)
+const consumer = new Kafka.SimpleConsumer(options)
 
 // data handler
-const dataHandler = async (messageSet, topic, partition) => Promise.each(messageSet, async (m) => {
+const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, (m) => {
   const message = m.message.value.toString('utf8')
   logger.info(`Handle Kafka event message; Topic: ${topic}; Partition: ${partition}; Offset: ${
     m.offset}; Message: ${message}.`)
@@ -46,7 +47,7 @@ const dataHandler = async (messageSet, topic, partition) => Promise.each(message
     return
   }
 
-  try {
+  return (async () => {
     switch (topic) {
       case config.CREATE_DATA_TOPIC:
         await ProcessorService.upsert(messageJSON)
@@ -60,12 +61,10 @@ const dataHandler = async (messageSet, topic, partition) => Promise.each(message
       default:
         throw new Error(`Invalid topic: ${topic}`)
     }
-
+  })()
     // commit offset
-    await consumer.commitOffset({ topic, partition, offset: m.offset })
-  } catch (err) {
-    logger.logFullError(err)
-  }
+    .then(() => consumer.commitOffset({ topic, partition, offset: m.offset }))
+    .catch((err) => logger.logFullError(err))
 })
 
 // check if there is kafka connection alive
@@ -81,15 +80,16 @@ function check () {
   return connected
 }
 
-const topics = [config.CREATE_DATA_TOPIC, config.UPDATE_DATA_TOPIC, config.DELETE_DATA_TOPIC]
-// consume configured topics
 consumer
-  .init([{
-    subscriptions: topics,
-    handler: dataHandler
-  }])
+  .init()
+  // consume configured topics
   .then(() => {
     healthcheck.init([check])
+
+    const topics = [config.CREATE_DATA_TOPIC, config.UPDATE_DATA_TOPIC, config.DELETE_DATA_TOPIC]
+    _.each(topics, (tp) => {
+      consumer.subscribe(tp, { time: Kafka.LATEST_OFFSET }, dataHandler)
+    })
   })
   .catch((err) => logger.logFullError(err))
 

src/common/helper.js

@@ -7,7 +7,7 @@ const _ = require('lodash')
 const request = require('superagent')
 const m2mAuth = require('tc-core-library-js').auth.m2m
 
-const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME']))
+const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL']))
 
 /*
  * Check if the Group ID is configured to be processed

src/services/ProcessorService.js

@@ -14,7 +14,7 @@ const {
  * Returns the tests passed using the metadata information
  * @param {object} metadata the object from which to retrieve the tests passed
  */
-function getTestsPassed (metadata = {}) {
+function getTestsPassed(metadata = {}) {
   const tests = metadata.assertions || {}
 
   let testsPassed = tests.total - tests.pending - tests.failed
@@ -30,7 +30,7 @@ function getTestsPassed (metadata = {}) {
  * Handle create / update topic messages from Kafka queue
  * @param {Object} message the Kafka message in JSON format
  */
-const upsert = async (message) => {
+const upsert = async(message) => {
   const submission = await helper.reqToAPI(`${config.SUBMISSION_API_URL}/${message.payload.submissionId}`)
 
   const existRecord = await Leaderboard.findOne({
@@ -71,7 +71,7 @@ const upsert = async (message) => {
 
     if (!helper.isGroupIdValid(challengeDetail.body.result.content[0].groupIds)) {
       logger.debug(`Group ID of Challenge # ${submission.body.challengeId} is not configured for processing!`)
-      // Ignore the message
+        // Ignore the message
       return
     }
 
@@ -107,7 +107,7 @@ upsert.schema = {
  * Handle delete topic message from Kafka Queue
  * @param {Object} message the Kafka message in JSON format
  */
-const remove = async (message) => {
+const remove = async(message) => {
   // Remove the record from MongoDB
   await Leaderboard.deleteOne({
     reviewSummationId: message.payload.id