Merge pull request #21 from topcoder-platform/Issue_20

Issue 20

Author: cwdcwd

README.md

@@ -2,7 +2,7 @@
 
 ## Dependencies
 
-- nodejs https://nodejs.org/en/ (v10)
+- Nodejs (v10)
 - Kafka (v2)
 
 ## Configuration
@@ -35,6 +35,7 @@ The following parameters can be set in config files or in env variables:
 Also note that there is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
 
 Configuration for the tests is at `config/test.js`, only add such new configurations different from `config/default.js`
+
 - WAIT_TIME: wait time used in test, default is 1000 or one second
 - LEADERBOARD_API_URL: Leaderboard API URL used in testing
 - SUBMISSION_API_URL: Submission API URL used in testing
@@ -67,7 +68,7 @@ Configuration for the tests is at `config/test.js`, only add such new configurat
 - In the console, write some message, one message per line:
 
 ```bash
-  { "topic":"submission.notification.create", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"reviewSummation", "id": "d24d4180-65aa-42ec-a945-5fd21dec0508", "submissionId": "a34e1158-2c27-4d38-b079-5e5cca1bdcf7", "aggregateScore": 88, "scoreCardId": "b25a4180-65aa-42ec-a945-5fd21dec0503", "isPassing": true, "created": "2018-05-20T07:00:30.123Z", "updated": "2018-06-01T07:36:28.178Z", "createdBy": "admin", "updatedBy": "admin" } }
+  {"topic":"submission.notification.create","originator":"submission-api","timestamp":"2018-08-06T15:46:05.575Z","mime-type":"application/json","payload":{"resource":"review","id":"49871146-eb0a-4d0e-ab9a-adc94018c5da","submissionId":"6ff0c009-51ee-4c8e-aa0d-159c20503cc2","score":-1,"scoreCardId":30001852,"metadata":{"testType":"provisional","tests":{"pending":0,"failed":1,"total":10}},"created":"2019-11-06T15:02:35.539Z","updated":"2019-11-06T15:02:35.539Z","createdBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","updatedBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","status":"completed","reviewerId":"0301619c-3d9e-44c3-85cb-c20311100f7f","typeId":"52c91e85-745f-4e62-b592-9879a2dfe9fd"}}
 ```
 
 - optionally, use another terminal, go to same directory, start a consumer to view the messages:
@@ -115,78 +116,52 @@ npm start
 
 2. Refer `README.md` in `leaderboard-api` to start leaderboard api, all operations are under `leaderboard-api` project root folder
 
-- Start mock app and it will listen on 3001 PORT.
-
-```bash
-npm run mock-api
-```
-
-- Ensure you have start MongoDB and properly configure `MONGODB_URL`. Run the following commands to clear and insert test data, step up environment variables and start the app(it will listen on 3002 PORT!).
-
-```bash
-npm run init-db
-npm run test-data
-export CHALLENGE_API_URL=http://localhost:3001/challenges
-export MEMBER_API_URL=http://localhost:3001/users
-export PORT=3002
-npm start
-```
+3. Set the necessary environment variables and then run `npm start` command to start processor app(Under this project's root folder)
 
-3. Run the following command to start processor app(Under this project's root folder)
-```bash
-export AUTH0_URL=https://topcoder-dev.auth0.com/oauth/token
-export AUTH0_AUDIENCE=https://m2m.topcoder-dev.com/
-export TOKEN_CACHE_TIME=90
-export AUTH0_CLIENT_ID=8QovDh27SrDu1XSs68m21A1NBP8isvOt
-export AUTH0_CLIENT_SECRET=3QVxxu20QnagdH-McWhVz0WfsQzA1F8taDdGDI4XphgpEYZPcMTF4lX3aeOIeCzh
-export LEADERBOARD_API_URL=http://localhost:3002/v5/leaderboard
-export SUBMISSION_API_URL=http://localhost:3001/submissions
-npm start
-```
+4. Attach to the topic `submission.notification.create` using Kafka console producer
 
-3. Attach to the topic `submission.notification.create` using Kafka console producer
+    ```bash
+    bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.create
+    ```
 
-```bash
-bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.create
-```
+5. Write the following message to the Console
 
-4. Write the following message to the Console
+    ```bash
+    {"topic":"submission.notification.create","originator":"submission-api","timestamp":"2018-08-06T15:46:05.575Z","mime-type":"application/json","payload":{"resource":"review","id":"49871146-eb0a-4d0e-ab9a-adc94018c5da","submissionId":"6ff0c009-51ee-4c8e-aa0d-159c20503cc2","score":-1,"scoreCardId":30001852,"metadata":{"testType":"provisional","tests":{"pending":0,"failed":1,"total":10}},"created":"2019-11-06T15:02:35.539Z","updated":"2019-11-06T15:02:35.539Z","createdBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","updatedBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","status":"completed","reviewerId":"0301619c-3d9e-44c3-85cb-c20311100f7f","typeId":"52c91e85-745f-4e62-b592-9879a2dfe9fd"}}
+    ```
 
-```bash
-{ "topic":"submission.notification.create", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"reviewSummation", "id": "d24d4180-65aa-42ec-a945-5fd21dec0508", "submissionId": "a34e1158-2c27-4d38-b079-5e5cca1bdcf7", "aggregateScore": 90, "scoreCardId": "b25a4180-65aa-42ec-a945-5fd21dec0503", "isPassing": true, "metadata": { "assertions": { "pending": 0, "failed": 1, "total": 10}, "tests": { "total": 10 } }, "created": "2018-05-20T07:00:30.123Z", "updated": "2018-06-01T07:36:28.178Z", "createdBy": "admin", "updatedBy": "admin" } }
-```
+6. You could see in the console that message will be processed, and find the following message: `Record with Challenge ID # 30051825 and Member ID # 8547899 does not exists in database. Creating the record`. Also check the leaderboard-api console for more information(Console in step 2)
 
-5. You could see in the console that message will be processed, and find the following message: `Record with Challenge ID # 30051825 and Member ID # 8547899 does not exists in database. Creating the record`. Also check the leaderboard-api console for more information(Console in step 2)
+7. Attach to the topic `submission.notification.update` using Kafka console producer
 
-6. Attach to the topic `submission.notification.update` using Kafka console producer
+    ```bash
+    bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.update
+    ```
 
-```bash
-bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.update
-```
+8. Write the following message to the Console
 
-7. Write the following message to the Console
+    ```bash
+    {"topic":"submission.notification.update","originator":"submission-api","timestamp":"2018-08-06T15:46:05.575Z","mime-type":"application/json","payload":{"resource":"review","id":"49871146-eb0a-4d0e-ab9a-adc94018c5da","submissionId":"6ff0c009-51ee-4c8e-aa0d-159c20503cc2","score":52,"scoreCardId":30001852,"metadata":{"testType":"provisional","tests":{"pending":0,"failed":2,"total":8}},"created":"2019-11-06T15:02:35.539Z","updated":"2019-11-06T15:02:35.539Z","createdBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","updatedBy":"I3etJtTqlz1XHgCXduBN1us705ufrykl@clients","status":"completed","reviewerId":"0301619c-3d9e-44c3-85cb-c20311100f7f","typeId":"52c91e85-745f-4e62-b592-9879a2dfe9fd"}}
+    ```
 
-```bash
-{ "topic":"submission.notification.update", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"reviewSummation", "id": "d24d4180-65aa-42ec-a945-5fd21dec0508", "submissionId": "a34e1158-2c27-4d38-b079-5e5cca1bdcf7", "aggregateScore": 80, "scoreCardId": "b25a4180-65aa-42ec-a945-5fd21dec0503", "isPassing": true, "metadata": { "assertions": { "pending": 0, "failed": 1, "total": 5}, "tests": { "total": 5 } }, "created": "2018-05-20T07:00:30.123Z", "updated": "2018-06-01T07:36:28.178Z", "createdBy": "admin", "updatedBy": "admin" } }
-```
+9. You could see in the console that message will be processed, and find the following message: `Record with Challenge ID # 30051825 and Member ID # 8547899 exists in database. Updating the score`. Also check the leaderboard-api console for more information(Console in step 2)
 
-8. You could see in the console that message will be processed, and find the following message: `Record with Challenge ID # 30051825 and Member ID # 8547899 exists in database. Updating the score`. Also check the leaderboard-api console for more information(Console in step 2)
+10. Attach to the topic `submission.notification.delete` using Kafka console producer
 
-9. Attach to the topic `submission.notification.delete` using Kafka console producer
+    ```bash
+    bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.delete
+    ```
 
-```bash
-bin/kafka-console-producer.sh --broker-list localhost:9092 --topic submission.notification.delete
-```
+11. Write the following message to the Console. Also check the leaderboard-api console for more information(Console in step 2)
 
-10. Write the following message to the Console. Also check the leaderboard-api console for more information(Console in step 2)
+    ```bash
+    { "topic":"submission.notification.delete", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"review", "id": "49871146-eb0a-4d0e-ab9a-adc94018c5da" } }
+    ```
 
-```bash
-{ "topic":"submission.notification.delete", "originator":"submission-api", "timestamp":"2018-08-06T15:46:05.575Z", "mime-type":"application/json", "payload":{ "resource":"reviewSummation", "id": "d24d4180-65aa-42ec-a945-5fd21dec0508" } }
-```
+12. You could see in the console that message will be processed
 
-11. You could see in the console that message will be processed
+## Tests
 
-### Tests
 Note: you need to stop the processor app before execute test.
 
 - Run the following command to execute unit test and generate coverage report

config/default.js

@@ -21,6 +21,7 @@ module.exports = {
 
   LEADERBOARD_API_URL: process.env.LEADERBOARD_API_URL || 'https://api.topcoder-dev.com/v5/leaderboard',
   SUBMISSION_API_URL: process.env.SUBMISSION_API_URL || 'https://api.topcoder-dev.com/v5/submissions',
+  REVIEW_TYPE_URL: process.env.REVIEW_TYPE_URL || 'https://api.topcoder-dev.com/v5/reviewTypes',
 
   AUTH0_URL: process.env.AUTH0_URL,
   AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,

src/app.js

@@ -8,6 +8,7 @@ const logger = require('./common/logger')
 const Kafka = require('no-kafka')
 const healthcheck = require('topcoder-healthcheck-dropin')
 const ProcessorService = require('./services/ProcessorService')
+const helper = require('./common/helper')
 
 // start Kafka consumer
 logger.info('Start Kafka consumer.')
@@ -19,7 +20,7 @@ if (config.KAFKA_CLIENT_CERT && config.KAFKA_CLIENT_CERT_KEY) {
 const consumer = new Kafka.GroupConsumer(options)
 
 // data handler
-const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, (m) => {
+const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, async (m) => {
   const message = m.message.value.toString('utf8')
   logger.info(`Handle Kafka event message; Topic: ${topic}; Partition: ${partition}; Offset: ${
     m.offset}; Message: ${message}.`)
@@ -40,12 +41,19 @@ const dataHandler = (messageSet, topic, partition) => Promise.each(messageSet, (
     return
   }
 
-  if (messageJSON.payload.resource !== 'reviewSummation') {
-    logger.debug(`Ignoring Non review Summation payloads from topic ${messageJSON.topic}.`)
+  if (messageJSON.payload.resource !== 'review') {
+    logger.debug(`Ignoring Non review payloads from topic ${messageJSON.topic}.`)
     // ignore the message
     return
   }
 
+  const avScanTypeId = await helper.getreviewTypeId('AV Scan')
+
+  if (messageJSON.payload.typeId === avScanTypeId) {
+    logger.debug(`Ignoring AV Scan reviews from topic ${messageJSON.topic}`)
+    return false
+  }
+
   return (async () => {
     switch (topic) {
       case config.CREATE_DATA_TOPIC:

src/common/helper.js

@@ -9,6 +9,9 @@ const m2mAuth = require('tc-core-library-js').auth.m2m
 
 const m2m = m2mAuth(_.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL']))
 
+// Variable to cache reviewTypes from Submission API
+const reviewTypes = {}
+
 /*
  * Function to get M2M token
  * @returns {Promise}
@@ -55,6 +58,25 @@ const reqToAPI = async (reqType, path, reqBody) => {
   })
 }
 
+/*
+ * Function to get reviewTypeId from Name
+ * @param {String} reviewTypeName Name of the reviewType
+ * @returns {String} reviewTypeId
+ */
+const getreviewTypeId = async (reviewTypeName) => {
+  if (!reviewTypes[reviewTypeName]) {
+    // Get review type id from Submission API
+    const response = await reqToAPI('GET', `${config.REVIEW_TYPE_URL}?name=${reviewTypeName}`)
+    if (response.body && response.body.length !== 0) {
+      reviewTypes[reviewTypeName] = response.body[0].id
+    } else {
+      reviewTypes[reviewTypeName] = null
+    }
+  }
+  return reviewTypes[reviewTypeName]
+}
+
 module.exports = {
-  reqToAPI
+  reqToAPI,
+  getreviewTypeId
 }

src/services/ProcessorService.js

@@ -42,7 +42,7 @@ upsert.schema = {
  * @param {Object} message the Kafka message in JSON format
  */
 async function remove (message) {
-  await helper.reqToAPI('DELETE', `${config.LEADERBOARD_API_URL}/reviewSummation/${message.payload.id}`)
+  await helper.reqToAPI('DELETE', `${config.LEADERBOARD_API_URL}/review/${message.payload.id}`)
 }
 
 remove.schema = {

test/common/prepare.js

@@ -4,7 +4,7 @@
 
 const nock = require('nock')
 const prepare = require('mocha-prepare')
-const { submissionAPIResponse } = require('./testData')
+const { submissionAPIResponse, reviewTypesResponse } = require('./testData')
 
 prepare(function (done) {
   nock(/\.com/)
@@ -25,8 +25,10 @@ prepare(function (done) {
     .reply(200)
     .patch('/v5/leaderboard/challenge/30051825/member/22688726')
     .reply(200)
-    .delete('/v5/leaderboard/reviewSummation/d24d4180-65aa-42ec-a945-5fd21dec0510')
+    .delete('/v5/leaderboard/review/49871146-eb0a-4d0e-ab9a-adc94018c5da')
     .reply(204)
+    .get('/v5/reviewTypes?name=AV%20Scan')
+    .reply(200, reviewTypesResponse)
 
   done()
 }, function (done) {

test/common/testData.js

@@ -9,19 +9,29 @@ const testTopics = {
     testMessage: {
       topic: 'submission.notification.create',
       originator: 'submission-api',
-      timestamp: '2018-02-03T00:00:00',
+      timestamp: '2018-08-06T15:46:05.575Z',
       'mime-type': 'application/json',
       payload: {
-        resource: 'reviewSummation',
-        id: 'd24d4180-65aa-42ec-a945-5fd21dec0508',
+        resource: 'review',
+        id: '49871146-eb0a-4d0e-ab9a-adc94018c5da',
         submissionId: 'a34e1158-2c27-4d38-b079-5e5cca1bdcf7',
-        aggregateScore: 88,
-        scoreCardId: 'b25a4180-65aa-42ec-a945-5fd21dec0503',
-        isPassing: true,
-        created: '2018-01-01T00:00:00',
-        updated: '2018-01-02T00:00:00',
-        createdBy: 'admin',
-        updatedBy: 'user'
+        score: -1,
+        scoreCardId: 30001852,
+        metadata: {
+          testType: 'provisional',
+          tests: {
+            pending: 0,
+            failed: 1,
+            total: 10
+          }
+        },
+        created: '2019-11-06T15:02:35.539Z',
+        updated: '2019-11-06T15:02:35.539Z',
+        createdBy: 'I3etJtTqlz1XHgCXduBN1us705ufrykl@clients',
+        updatedBy: 'I3etJtTqlz1XHgCXduBN1us705ufrykl@clients',
+        status: 'completed',
+        reviewerId: '0301619c-3d9e-44c3-85cb-c20311100f7f',
+        typeId: '52c91e85-745f-4e62-b592-9879a2dfe9fd'
       }
     }
   },
@@ -31,14 +41,29 @@ const testTopics = {
     testMessage: {
       topic: 'submission.notification.update',
       originator: 'submission-api',
-      timestamp: '2018-02-03T00:00:00',
+      timestamp: '2018-08-06T15:46:05.575Z',
       'mime-type': 'application/json',
       payload: {
-        resource: 'reviewSummation',
-        id: 'd24d4180-65aa-42ec-a945-5fd21dec0509',
+        resource: 'review',
+        id: '49871146-eb0a-4d0e-ab9a-adc94018c5da',
         submissionId: 'a34e1158-2c27-4d38-b079-5e5cca1bdcf8',
-        aggregateScore: 93.5,
-        isPassing: true
+        score: 52,
+        scoreCardId: 30001852,
+        metadata: {
+          testType: 'provisional',
+          tests: {
+            pending: 0,
+            failed: 2,
+            total: 10
+          }
+        },
+        created: '2019-11-06T15:02:35.539Z',
+        updated: '2019-11-06T15:02:35.539Z',
+        createdBy: 'I3etJtTqlz1XHgCXduBN1us705ufrykl@clients',
+        updatedBy: 'I3etJtTqlz1XHgCXduBN1us705ufrykl@clients',
+        status: 'completed',
+        reviewerId: '0301619c-3d9e-44c3-85cb-c20311100f7f',
+        typeId: '52c91e85-745f-4e62-b592-9879a2dfe9fd'
       }
     }
   },
@@ -51,8 +76,8 @@ const testTopics = {
       timestamp: '2018-02-03T00:00:00',
       'mime-type': 'application/json',
       payload: {
-        resource: 'reviewSummation',
-        id: 'd24d4180-65aa-42ec-a945-5fd21dec0510'
+        resource: 'review',
+        id: '49871146-eb0a-4d0e-ab9a-adc94018c5da'
       }
     }
   }
@@ -79,7 +104,7 @@ const submissionAPIResponse = [
     'created': '2018-08-22T02:41:07.576Z',
     'isFileSubmission': true,
     'type': 'challengesubmission',
-    'url': 'https://topcoder-dev-submissions-dmz.s3.amazonaws.com/a34e1158-2c27-4d38-b079-%3Cspan%20class="x x-first x-last">5e5cca1bdcf7',
+    'url': 'https://topcoder-dev-submissions-dmz.s3.amazonaws.com/a34e1158-2c27-4d38-b079-%3Cspan%20class="x x-first x-last">5e5cca1bdcf8',
     'challengeId': 30051825,
     'filename': 'CORE-TopcoderEventBus-190618-1413.pdf.zip',
     'createdBy': 'lazybaer',
@@ -91,7 +116,16 @@ const submissionAPIResponse = [
   }
 ]
 
+const reviewTypesResponse = [
+  {
+    'name': 'AV Scan',
+    'id': '6da98d0f-e663-4539-8507-cd6c9e0e56d8',
+    'isActive': true
+  }
+]
+
 module.exports = {
   submissionAPIResponse,
-  testTopics
+  testTopics,
+  reviewTypesResponse
 }

test/e2e/test.js

@@ -224,7 +224,7 @@ describe('Topcoder - Leaderboard Processor E2E Test', () => {
     await waitJob()
 
     expect(errorLogs.length).to.equal(0)
-    expect(debugLogs[debugLogs.length - 1]).to.equal(`Ignoring Non review Summation payloads from topic ${message.topic}.`)
+    expect(debugLogs[debugLogs.length - 1]).to.equal(`Ignoring Non review payloads from topic ${message.topic}.`)
   })
 
   for (const op of ['create', 'update', 'delete']) {