From f3f01176ebc1fa8a77fdb8d8ff5dac3430b9e65e Mon Sep 17 00:00:00 2001 From: Thomas Kranitsas Date: Fri, 25 Jan 2019 09:11:01 +0200 Subject: [PATCH 1/3] Revert "Fix bug in processor" This reverts commit d81eaddfce8481fbe81202195a3d4f25bee7ebd2. --- README.md | 1 - config/default.js | 1 - package-lock.json | 29 ++++++++++++++++++----------- package.json | 2 +- src/app.js | 30 +++++++++++++++--------------- src/services/ProcessorService.js | 8 ++++---- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index d1fa1aa..d4082fe 100755 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ The following parameters can be set in config files or in env variables: - LOG_LEVEL: the log level - PORT: the server port -- KAFKA_GROUP_ID: group id of the consumer; default value: 'leaderboard-processor-group' - KAFKA_URL: comma separated Kafka hosts - KAFKA_CLIENT_CERT: Kafka connection certificate, optional; if not provided, then SSL connection is not used, direct insecure connection is used; diff --git a/config/default.js b/config/default.js index 90c7317..7f4ef0c 100755 --- a/config/default.js +++ b/config/default.js @@ -6,7 +6,6 @@ module.exports = { LOG_LEVEL: process.env.LOG_LEVEL || 'debug', PORT: process.env.PORT || 3000, - KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'leaderboard-processor-group', KAFKA_URL: process.env.KAFKA_URL || 'localhost:9092', // below two params are used for secure Kafka connection, they are optional // for the local Kafka, they are not needed diff --git a/package-lock.json b/package-lock.json index 2f8cf7f..668d87a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -178,9 +178,9 @@ } }, "@types/lodash": { - "version": "4.14.119", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.119.tgz", - "integrity": "sha512-Z3TNyBL8Vd/M9D9Ms2S3LmFq2sSMzahodD6rCS9V2N44HUMINb75jNkSuwAx7eo2ufqTdfOdtGQpNbieUjPQmw==" + "version": "4.14.117", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.117.tgz", + "integrity": "sha512-xyf2m6tRbz8qQKcxYZa7PA4SllYcay+eh25DN3jmNYY6gSTL7Htc/bttVdkqj2wfJGbeWlQiX8pIyJpKU+tubw==" }, "@types/mime": { "version": "2.0.0", @@ -2586,9 +2586,9 @@ "dev": true }, "no-kafka": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.4.3.tgz", - "integrity": "sha512-hYnkg1OWVdaxORdzVvdQ4ueWYpf7IICObPzd24BBiDyVG5219VkUnRxSH9wZmisFb6NpgABzlSIL1pIZaCKmXg==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/no-kafka/-/no-kafka-3.3.2.tgz", + "integrity": "sha512-A9YZaItJeMK46zi3ePS1T0HS5YHiNjggRrpCaea8GDqwmglVdP3WP40wY+cFKwLl851BUUP0LrFi60mdy9hfdg==", "requires": { "@types/bluebird": "3.5.0", "@types/lodash": "^4.14.55", @@ -2596,10 +2596,17 @@ "bluebird": "^3.3.3", "buffer-crc32": "^0.2.5", "hashring": "^3.2.0", - "lodash": "=4.17.11", + "lodash": "=4.17.5", "murmur-hash-js": "^1.0.0", "nice-simple-logger": "^1.0.1", "wrr-pool": "^1.0.3" + }, + "dependencies": { + "lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==" + } } }, "nock": { @@ -5849,11 +5856,11 @@ } }, "wrr-pool": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.4.tgz", - "integrity": "sha512-+lEdj42HlYqmzhvkZrx6xEymj0wzPBxqr7U1Xh9IWikMzOge03JSQT9YzTGq54SkOh/noViq32UejADZVzrgAg==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wrr-pool/-/wrr-pool-1.1.3.tgz", + "integrity": "sha1-/a0i8uofMDY//l14HPeUl6d/8H4=", "requires": { - "lodash": "^4.17.11" + "lodash": "^4.0.1" } }, "xtend": { diff --git a/package.json b/package.json index cbb1f6d..2314847 100755 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "joi": "^14.0.0", "lodash": "^4.17.11", "mongoose": "^5.3.3", - "no-kafka": "^3.4.3", + "no-kafka": "^3.2.4", "superagent": "^3.8.3", "tc-core-library-js": "appirio-tech/tc-core-library-js.git#feature/m2mtoken", "topcoder-healthcheck-dropin": "^1.0.3", diff --git a/src/app.js b/src/app.js index 9fa7eb7..d532936 100755 --- a/src/app.js +++ b/src/app.js @@ -4,6 +4,7 @@ require('./bootstrap') const config = require('config') +const _ = require('lodash') const logger = require('./common/logger') const Kafka = require('no-kafka') const healthcheck = require('topcoder-healthcheck-dropin') @@ -12,14 +13,14 @@ const ProcessorService = require('./services/ProcessorService') // start Kafka consumer logger.info('Start Kafka consumer.') // create consumer -const options = { connectionString: config.KAFKA_URL, groupId: config.KAFKA_GROUP_ID } +const options = { connectionString: config.KAFKA_URL } if (config.KAFKA_CLIENT_CERT && config.KAFKA_CLIENT_CERT_KEY) { options.ssl = { cert: config.KAFKA_CLIENT_CERT, key: config.KAFKA_CLIENT_CERT_KEY } } -const consumer = new Kafka.GroupConsumer(options) +const consumer = new Kafka.SimpleConsumer(options) // data handler -const dataHandler = async (messageSet, topic, partition) => Promise.each(messageSet, async (m) => { +const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, (m) => { const message = m.message.value.toString('utf8') logger.info(`Handle Kafka event message; Topic: ${topic}; Partition: ${partition}; Offset: ${ m.offset}; Message: ${message}.`) @@ -46,7 +47,7 @@ const dataHandler = async (messageSet, topic, partition) => Promise.each(message return } - try { + return (async () => { switch (topic) { case config.CREATE_DATA_TOPIC: await ProcessorService.upsert(messageJSON) @@ -60,12 +61,10 @@ const dataHandler = async (messageSet, topic, partition) => Promise.each(message default: throw new Error(`Invalid topic: ${topic}`) } - + })() // commit offset - await consumer.commitOffset({ topic, partition, offset: m.offset }) - } catch (err) { - logger.logFullError(err) - } + .then(() => consumer.commitOffset({ topic, partition, offset: m.offset })) + .catch((err) => logger.logFullError(err)) }) // check if there is kafka connection alive @@ -81,15 +80,16 @@ function check () { return connected } -const topics = [config.CREATE_DATA_TOPIC, config.UPDATE_DATA_TOPIC, config.DELETE_DATA_TOPIC] -// consume configured topics consumer - .init([{ - subscriptions: topics, - handler: dataHandler - }]) + .init() + // consume configured topics .then(() => { healthcheck.init([check]) + + const topics = [config.CREATE_DATA_TOPIC, config.UPDATE_DATA_TOPIC, config.DELETE_DATA_TOPIC] + _.each(topics, (tp) => { + consumer.subscribe(tp, { time: Kafka.LATEST_OFFSET }, dataHandler) + }) }) .catch((err) => logger.logFullError(err)) diff --git a/src/services/ProcessorService.js b/src/services/ProcessorService.js index 04deabc..a781e4e 100755 --- a/src/services/ProcessorService.js +++ b/src/services/ProcessorService.js @@ -14,7 +14,7 @@ const { * Returns the tests passed using the metadata information * @param {object} metadata the object from which to retrieve the tests passed */ -function getTestsPassed (metadata = {}) { +function getTestsPassed(metadata = {}) { const tests = metadata.assertions || {} let testsPassed = tests.total - tests.pending - tests.failed @@ -30,7 +30,7 @@ function getTestsPassed (metadata = {}) { * Handle create / update topic messages from Kafka queue * @param {Object} message the Kafka message in JSON format */ -const upsert = async (message) => { +const upsert = async(message) => { const submission = await helper.reqToAPI(`${config.SUBMISSION_API_URL}/${message.payload.submissionId}`) const existRecord = await Leaderboard.findOne({ @@ -71,7 +71,7 @@ const upsert = async (message) => { if (!helper.isGroupIdValid(challengeDetail.body.result.content[0].groupIds)) { logger.debug(`Group ID of Challenge # ${submission.body.challengeId} is not configured for processing!`) - // Ignore the message + // Ignore the message return } @@ -107,7 +107,7 @@ upsert.schema = { * Handle delete topic message from Kafka Queue * @param {Object} message the Kafka message in JSON format */ -const remove = async (message) => { +const remove = async(message) => { // Remove the record from MongoDB await Leaderboard.deleteOne({ reviewSummationId: message.payload.id From 76c593b80fba76e6ed7638733b14e9a1158aa221 Mon Sep 17 00:00:00 2001 From: Sachin Maheshwari Date: Mon, 18 Mar 2019 18:32:20 +0530 Subject: [PATCH 2/3] adding auth0 proxy server changes --- config/default.js | 1 + package.json | 2 +- src/common/helper.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/default.js b/config/default.js index 7f4ef0c..e5044e7 100755 --- a/config/default.js +++ b/config/default.js @@ -29,5 +29,6 @@ module.exports = { TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME, AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET, + AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL, MONGODB_URL: process.env.MONGODB_URL || 'mongodb://localhost:27017/leaderboardDB' } diff --git a/package.json b/package.json index 2314847..251ab65 100755 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "mongoose": "^5.3.3", "no-kafka": "^3.2.4", "superagent": "^3.8.3", - "tc-core-library-js": "appirio-tech/tc-core-library-js.git#feature/m2mtoken", + "tc-core-library-js": "appirio-tech/tc-core-library-js.git#v2.6", "topcoder-healthcheck-dropin": "^1.0.3", "winston": "^3.1.0" }, diff --git a/src/common/helper.js b/src/common/helper.js index 209643f..56b7e4b 100755 --- a/src/common/helper.js +++ b/src/common/helper.js @@ -7,7 +7,7 @@ const _ = require('lodash') const request = require('superagent') const m2mAuth = require('tc-core-library-js').auth.m2m -const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME'])) +const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])) /* * Check if the Group ID is configured to be processed From 2f85eb5b8a86ed248d7123127b24901e2a0c236c Mon Sep 17 00:00:00 2001 From: Gunasekar-K Date: Tue, 19 Mar 2019 16:53:37 +0530 Subject: [PATCH 3/3] STS integration --- .circleci/config.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1de75f1..5864da9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,10 +5,17 @@ defaults: &defaults install_dependency: &install_dependency name: Installation of build and deployment dependencies. command: | - apk add --no-cache bash + apk add --no-cache bash curl apk add --no-cache jq py-pip sudo sudo pip install awscli --upgrade sudo pip install docker-compose +install_deploysuite: &install_deploysuite + name: Installation of install_deploysuite. + command: | + git clone --branch v1.3 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript + cp ./../buildscript/master_deploy.sh . + cp ./../buildscript/buildenv.sh . + cp ./../buildscript/awsconfiguration.sh . restore_cache_settings_for_build: &restore_cache_settings_for_build key: docker-node-modules-lp-{{ checksum "package-lock.json" }} @@ -25,15 +32,21 @@ jobs: - checkout - setup_remote_docker - run: *install_dependency + - run: *install_deploysuite - restore_cache: *restore_cache_settings_for_build - run: ./build.sh - save_cache: *save_cache_settings - deploy: name: Running MasterScript. command: | - git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript - cp ./../buildscript/master_deploy.sh . - ./master_deploy.sh -d ECS -e DEV -t latest -s leaderboard-processor -i leaderboard-processor + #git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript + #cp ./../buildscript/master_deploy.sh . + #./master_deploy.sh -d ECS -e DEV -t latest -s leaderboard-processor -i leaderboard-processor + ./awsconfiguration.sh DEV + source awsenvconf + ./buildenv.sh -e DEV -b dev-leaderboard-processor-deployvar + source buildenvvar + ./master_deploy.sh -d ECS -e DEV -t latest -s dev-global-appvar,dev-leaderboard-processor-appvar -i leaderboard-processor # Build & Deploy against production backend "build-prod": @@ -42,15 +55,21 @@ jobs: - checkout - setup_remote_docker - run: *install_dependency + - run: *install_deploysuite - restore_cache: *restore_cache_settings_for_build - run: ./build.sh - save_cache: *save_cache_settings - deploy: name: Running MasterScript. command: | - git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript - cp ./../buildscript/master_deploy.sh . - ./master_deploy.sh -d ECS -e PROD -t latest -s leaderboard-processor -i leaderboard-processor + #git clone --branch v1.1 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript + #cp ./../buildscript/master_deploy.sh . + #./master_deploy.sh -d ECS -e PROD -t latest -s leaderboard-processor -i leaderboard-processor + ./awsconfiguration.sh PROD + source awsenvconf + ./buildenv.sh -e PROD -b prod-leaderboard-processor-deployvar + source buildenvvar + ./master_deploy.sh -d ECS -e PROD -t latest -s prod-global-appvar,prod-leaderboard-processor-appvar -i leaderboard-processor workflows: version: 2 @@ -58,12 +77,13 @@ workflows: jobs: # Development builds are executed on "develop" branch only. - "build-dev": + context : org-global filters: branches: only: - - circleci - develop - "build-prod": + context : org-global filters: branches: only: