diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e095a20..51b162b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,7 @@ workflows: only: - dev - CORE-40 + - CORE-154 - "build-qa": context: org-global diff --git a/app.js b/app.js index c72672f2..5b113e4d 100644 --- a/app.js +++ b/app.js @@ -7,6 +7,8 @@ require("./app-bootstrap"); const _ = require("lodash"); const config = require("config"); const express = require("express"); +const compressible = require("compressible"); +const compression = require("compression"); const bodyParser = require("body-parser"); const cors = require("cors"); const HttpStatus = require("http-status-codes"); @@ -20,6 +22,20 @@ const { ForbiddenError } = require("./src/common/errors"); // setup express app const app = express(); +app.use( + compression({ + filter: (req, res) => { + var type = res.getHeader("Content-Type"); + + if (!compressible(type)) { + return false; + } + + // enable compression for /challenges/topgear api + return req.path && req.path.startsWith(`/${config.API_VERSION}/challenges/topgear`); + }, + }) +); // Disable POST, PUT, PATCH, DELETE operations if READONLY is set to true app.use((req, res, next) => { diff --git a/config/default.js b/config/default.js index c0a71032..d3697af3 100644 --- a/config/default.js +++ b/config/default.js @@ -49,6 +49,8 @@ module.exports = { TEMP_REINDEXING: process.env.TEMP_REINDEXING || true, // if true, it won't delete the existing index when reindexing data }, + CENTRAL_SEARCH_URL: process.env.CENTRAL_SEARCH_URL || "https://vpc-centra-search-glnpbuur5r2ffx7fzcnkwzzcfi.us-east-1.es.amazonaws.com/", + // in bytes FILE_UPLOAD_SIZE_LIMIT: process.env.FILE_UPLOAD_SIZE_LIMIT ? Number(process.env.FILE_UPLOAD_SIZE_LIMIT) @@ -73,6 +75,8 @@ module.exports = { ? process.env.COPILOT_RESOURCE_ROLE_IDS.split(",") : ["10ba038e-48da-487b-96e8-8d3b99b6d18b"], SUBMITTER_ROLE_ID: process.env.SUBMITTER_ROLE_ID || "732339e7-8e30-49d7-9198-cccf9451e221", + REVIEWER_RESOURCE_ROLE_ID: process.env.REVIEWER_RESOURCE_ROLE_ID || '318b9c07-079a-42d9-a81f-b96be1dc1099', + ITERATIVE_REVIEWER_RESOURCE_ROLE_ID: process.env.ITERATIVE_REVIEWER_RESOURCE_ROLE_ID || 'f6df7212-b9d6-4193-bfb1-b383586fce63', MANAGER_ROLE_ID: process.env.MANAGER_ROLE_ID || "0e9c6879-39e4-4eb6-b8df-92407890faf1", OBSERVER_ROLE_ID: process.env.OBSERVER_ROLE_ID || "2a4dc376-a31c-4d00-b173-13934d89e286", diff --git a/package.json b/package.json index 2c702dbe..f224f2a0 100644 --- a/package.json +++ b/package.json @@ -42,8 +42,8 @@ "dependencies": { "@grpc/grpc-js": "^1.8.12", "@opensearch-project/opensearch": "^2.2.0", - "@topcoder-framework/domain-challenge": "^0.24.1", "@topcoder-framework/domain-acl": "^0.24.0", + "@topcoder-framework/domain-challenge": "^0.24.1", "@topcoder-framework/lib-common": "^0.24.1", "aws-sdk": "^2.1145.0", "axios": "^0.19.0", @@ -51,6 +51,7 @@ "bluebird": "^3.5.1", "body-parser": "^1.15.1", "compare-versions": "^6.1.0", + "compression": "^1.7.4", "config": "^3.0.1", "cors": "^2.8.5", "decimal.js": "^10.4.3", diff --git a/src/controllers/TopgearController.js b/src/controllers/TopgearController.js new file mode 100644 index 00000000..3ee4938a --- /dev/null +++ b/src/controllers/TopgearController.js @@ -0,0 +1,52 @@ +/** + * Controller for Topgear. + */ +const service = require("../services/TopgearService"); + +/** + * Get Topgear trending technologies. + * @param {Object} req the request + * @param {Object} res the response + */ +async function getTechTrending(req, res) { + const result = await service.getTechTrending(req.authUser, { + ...req.query, + ...req.body, + }); + res.send(result); +} + +/** + * Get Topgear member badges. + * @param {Object} req the request + * @param {Object} res the response + */ +async function getMemberBadges(req, res) { + const result = await service.getMemberBadges(req.authUser, { + ...req.query, + ...req.body, + }); + res.send(result); +} + +/** + * Search Topgear challenges. + * @param {Object} req the request + * @param {Object} res the response + */ +async function searchChallenges(req, res) { + const result = await service.searchChallenges(req.authUser, { + ...req.query, + ...req.body, + }); + if (result.cursor) { + res.header("X-Cursor", result.cursor); + } + res.send(result.challenges); +} + +module.exports = { + getTechTrending, + getMemberBadges, + searchChallenges, +}; diff --git a/src/routes.js b/src/routes.js index 8d46e69d..1ab468fa 100644 --- a/src/routes.js +++ b/src/routes.js @@ -34,6 +34,39 @@ module.exports = { scopes: [CREATE, ALL], }, }, + "/challenges/topgear": { + get: { + controller: "TopgearController", + method: "searchChallenges", + auth: "jwt", + access: [ + constants.UserRoles.Admin, + ], + scopes: [READ, ALL], + }, + }, + "/challenges/topgear/tech-trending": { + get: { + controller: "TopgearController", + method: "getTechTrending", + auth: "jwt", + access: [ + constants.UserRoles.Admin, + ], + scopes: [READ, ALL], + }, + }, + "/challenges/topgear/badges": { + get: { + controller: "TopgearController", + method: "getMemberBadges", + auth: "jwt", + access: [ + constants.UserRoles.Admin, + ], + scopes: [READ, ALL], + }, + }, "/challenges/support-requests": { post: { controller: "SupportController", diff --git a/src/services/TopgearService.js b/src/services/TopgearService.js new file mode 100644 index 00000000..cae0faf3 --- /dev/null +++ b/src/services/TopgearService.js @@ -0,0 +1,473 @@ +/** + * This service provides operations for Topgear. + */ +const _ = require("lodash"); +const Joi = require("joi"); +const moment = require("moment"); +const config = require("config"); +const { Client: SearchClient } = require("@opensearch-project/opensearch"); + +const logger = require("../common/logger"); +const errors = require("../common/errors"); +const constants = require("../../app-constants"); + +const searchClient = new SearchClient({ + node: config.CENTRAL_SEARCH_URL, +}); + +const indexName = "topgear_challenge"; + +/** + * Get Topgear trending technologies. + * @param {Object} currentUser The user who perform operation + * @param {Object} criteria The search criteria + */ +async function getTechTrending(currentUser, criteria) { + // Default to get trending technologies in 1 year, client can pass afterCompleteDate to override + const result = await searchClient.search({ + index: indexName, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { "status.keyword": constants.challengeStatuses.Completed } }, + { + range: { + endDate: { + gte: criteria.afterCompleteDate + ? criteria.afterCompleteDate + : moment().subtract(1, "years").toISOString(), + }, + }, + }, + ], + }, + }, + aggs: { + teches: { + terms: { + field: "tags.keyword", + size: 100000, + }, + }, + }, + }, + }); + + const aggs = result.body.aggregations; + const teches = []; + + _.forEach(_.get(aggs, "teches.buckets"), (bucket) => { + teches.push({ + tech: bucket.key, + count: bucket.doc_count, + }); + }); + + return teches.sort((a, b) => b.count - a.count); +} + +getTechTrending.schema = { + currentUser: Joi.any(), + criteria: Joi.object({ + afterCompleteDate: Joi.date(), + }).unknown(true), +}; + +/** + * Get Topgear member badges. + * @param {Object} currentUser The user who perform operation + * @param {Object} criteria The search criteria + */ +async function getMemberBadges(currentUser, criteria) { + // Default to get member badges in 1 year, client can pass afterCompleteDate to override + const result = await searchClient.search({ + index: indexName, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { "status.keyword": constants.challengeStatuses.Completed } }, + { + range: { + endDate: { + gte: criteria.afterCompleteDate + ? criteria.afterCompleteDate + : moment().subtract(1, "years").toISOString(), + }, + }, + }, + ], + }, + }, + aggs: { + wins: { + terms: { + field: "winners.userId", + size: 1000000, + }, + }, + submissions: { + terms: { + field: "submissions.memberId", + size: 1000000, + }, + }, + resources: { + nested: { + path: "resources", + }, + aggs: { + registrants: { + filter: { + term: { + "resources.roleId": config.SUBMITTER_ROLE_ID, + }, + }, + aggs: { + count: { + multi_terms: { + terms: [ + { + field: "resources.memberId", + }, + { + field: "resources.memberHandle.keyword", + }, + ], + size: 1000000, + }, + }, + }, + }, + }, + }, + }, + }, + }); + + const aggs = result.body.aggregations; + const members = new Map(); + + _.forEach(_.get(aggs, "resources.registrants.count.buckets"), (bucket) => { + const memberId = `${bucket.key[0]}`; + const memberHandle = bucket.key[1]; + + members.set(memberId, { memberId, memberHandle, challenges: bucket.doc_count }); + }); + + _.forEach(_.get(aggs, "submissions.buckets"), (bucket) => { + const memberId = `${bucket.key}`; + const member = members.get(memberId); + if (member) { + member.submissions = bucket.doc_count; + } + }); + + _.forEach(_.get(aggs, "wins.buckets"), (bucket) => { + const memberId = `${bucket.key}`; + const member = members.get(memberId); + if (member) { + member.wins = bucket.doc_count; + } + }); + + return Array.from(members.values()).sort((a, b) => b.challenges - a.challenges); +} + +getMemberBadges.schema = { + currentUser: Joi.any(), + criteria: Joi.object({ + afterCompleteDate: Joi.date(), + }).unknown(true), +}; + +/** + * Search Topgear challenges + * @param {Object} currentUser The user who perform operation + * @param {Object} criteria The search criteria + */ +async function searchChallenges(currentUser, criteria) { + if (criteria.includeAll) { + // When include all data, max size window, and no need cursor + criteria.size = 10000; + delete criteria.cursor; + } else if (criteria.cursor) { + // Decode base64 cursor + try { + criteria.cursor = JSON.parse(Buffer.from(criteria.cursor, "base64").toString("utf8")); + } catch (err) { + throw new errors.BadRequestError("Bad cursor"); + } + } + + const should = []; + + // Default to find Active/Completed challenges + if (!criteria.status || !criteria.status.length) { + criteria.status = [constants.challengeStatuses.Active, constants.challengeStatuses.Completed]; + } + + if (criteria.status.includes(constants.challengeStatuses.Active)) { + const activeFilter = { + bool: { + filter: [], + }, + }; + should.push(activeFilter); + + // Default to find Active challenges in 90 days, client can pass afterActiveDate to override + activeFilter.bool.filter.push({ + term: { "status.keyword": constants.challengeStatuses.Active }, + }); + activeFilter.bool.filter.push({ + range: { + startDate: { + gte: criteria.afterActiveDate || moment().subtract(90, "days").toISOString(), + }, + }, + }); + + delete criteria.afterActiveDate; + criteria.status = _.filter(criteria.status, (s) => s !== constants.challengeStatuses.Active); + } + + if (criteria.status.includes(constants.challengeStatuses.Completed)) { + const completeFilter = { + bool: { + filter: [], + }, + }; + should.push(completeFilter); + + // Default to find Completed challenges in 90 days, client can pass afterCompleteDate to override + completeFilter.bool.filter.push({ + term: { "status.keyword": constants.challengeStatuses.Completed }, + }); + completeFilter.bool.filter.push({ + range: { + endDate: { + gte: criteria.afterCompleteDate || moment().subtract(90, "days").toISOString(), + }, + }, + }); + + delete criteria.afterCompleteDate; + criteria.status = _.filter(criteria.status, (s) => s !== constants.challengeStatuses.Completed); + } + + if (criteria.status.length) { + for (const status of criteria.status) { + should.push({ match_phrase: { status: status } }); + } + } + + const filters = []; + if (criteria.afterActiveDate) { + filters.push({ range: { startDate: { gte: criteria.afterActiveDate } } }); + } + if (criteria.afterCompleteDate) { + filters.push({ range: { endDate: { gte: criteria.afterCompleteDate } } }); + } + + const must = []; + if (criteria.name) { + must.push({ + match: { + name: { + query: criteria.name, + operator: "and", + }, + }, + }); + } + if (criteria.createdBy) { + must.push({ + match: { + createdBy: { + query: criteria.createdBy, + operator: "and", + }, + }, + }); + } + + const query = { + bool: {}, + }; + + if (should.length) { + query.bool.minimum_should_match = 1; + query.bool.should = should; + } + if (filters.length) { + query.bool.filter = filters; + } + if (must.length) { + query.bool.must = must; + } + + const { challenges, cursor } = await scroll( + query, + criteria.size, + criteria.includeAll, + criteria.cursor + ); + + for (const challenge of challenges) { + challenge.registrants = []; + challenge.reviewers = []; + + const resources = challenge.resources || []; + for (const resource of resources) { + if (resource.roleId === config.SUBMITTER_ROLE_ID) { + challenge.registrants.push({ + memberId: resource.memberId, + memberHandle: resource.memberHandle, + submitInd: false, + }); + } else if ( + resource.roleId === config.REVIEWER_RESOURCE_ROLE_ID || + resource.roleId === config.ITERATIVE_REVIEWER_RESOURCE_ROLE_ID + ) { + challenge.reviewers.push({ + memberId: resource.memberId, + memberHandle: resource.memberHandle, + }); + } + } + + const submissions = challenge.submissions || []; + + submissions.sort( + (a, b) => moment(b.submittedDate).valueOf() - moment(a.submittedDate).valueOf() + ); + + for (const registrant of challenge.registrants) { + const submission = submissions.find((sub) => `${sub.memberId}` === `${registrant.memberId}`); + if (submission) { + registrant.submitInd = true; + registrant.submittedDate = submission.submittedDate; + registrant.review = submission.review; + } + } + + delete challenge.resources; + delete challenge.submissions; + } + + return { challenges, cursor }; +} + +searchChallenges.schema = { + currentUser: Joi.any(), + criteria: Joi.object({ + name: Joi.string(), + createdBy: Joi.string(), + status: Joi.array().items(Joi.string().valid(_.values(constants.challengeStatuses))), + afterActiveDate: Joi.date(), + afterCompleteDate: Joi.date(), + includeAll: Joi.boolean().default(false), + cursor: Joi.string().base64({ paddingRequired: false }), + size: Joi.number().integer().min(1).max(10000).default(1000), + }).unknown(true), +}; + +const includeFields = [ + "id", + "directProjectId", + "name", + "status", + "created", + "createdBy", + "startDate", + "endDate", + "scheduledEndDate", + "tags", + "groups", + "winners", + "payments", + "resources", + "submissions.memberId", + "submissions.submittedDate", + "submissions.review.scoreCardId", + "submissions.review.score", +]; + +async function scroll(query, size, includeAll, cursor) { + console.info( + "Topgear query:", + JSON.stringify(query), + `size: ${size}, includeAll: ${includeAll}` + + (cursor ? `, cursor: ${JSON.stringify(cursor)}` : "") + ); + + const startTime = new Date().getTime(); + + const allHits = []; + + while (true) { + const result = await searchClient.search({ + index: indexName, + size, + timeout: "10s", + body: { + query, + _source: { + includes: includeFields, + }, + sort: [ + { + created: "desc", + }, + { + id: "desc", + }, + ], + ...(cursor ? { search_after: cursor } : {}), + }, + }); + + const hits = _.get(result, "body.hits.hits"); + + if (!hits || !hits.length) { + cursor = null; + break; + } else { + for (const hit of hits) { + allHits.push(hit._source); + } + if (hits.length < size) { + cursor = null; + break; + } else { + cursor = hits[hits.length - 1].sort; + } + } + + if (!includeAll) { + break; + } + } + + console.info( + `Search Topgear challenges finishes, count: ${allHits.length}, time spent: ${ + (new Date().getTime() - startTime) / 1000 + }s` + ); + + return { + challenges: allHits, + cursor: cursor ? Buffer.from(JSON.stringify(cursor), "utf8").toString("base64") : null, + }; +} + +module.exports = { + getTechTrending, + getMemberBadges, + searchChallenges, +}; + +logger.buildService(module.exports); diff --git a/yarn.lock b/yarn.lock index 431fa665..4b043870 100644 --- a/yarn.lock +++ b/yarn.lock @@ -424,7 +424,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -accepts@~1.3.8: +accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== @@ -800,6 +800,11 @@ busboy@^1.6.0: dependencies: streamsearch "^1.1.0" +bytes@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" + integrity sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw== + bytes@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" @@ -1013,6 +1018,26 @@ component-emitter@^1.2.0, component-emitter@^1.3.0: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.1.tgz#ef1d5796f7d93f135ee6fb684340b26403c97d17" integrity sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ== +compressible@~2.0.16: + version "2.0.18" + resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" + integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== + dependencies: + mime-db ">= 1.43.0 < 2" + +compression@^1.7.4: + version "1.7.4" + resolved "https://registry.yarnpkg.com/compression/-/compression-1.7.4.tgz#95523eff170ca57c29a0ca41e6fe131f41e5bb8f" + integrity sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ== + dependencies: + accepts "~1.3.5" + bytes "3.0.0" + compressible "~2.0.16" + debug "2.6.9" + on-headers "~1.0.2" + safe-buffer "5.1.2" + vary "~1.1.2" + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -2793,7 +2818,7 @@ millisecond@^0.1.2: resolved "https://registry.yarnpkg.com/millisecond/-/millisecond-0.1.2.tgz#6cc5ad386241cab8e78aff964f87028eec92dac5" integrity sha512-BJ8XtxY+woL+5TkP6uS6XvOArm0JVrX2otkgtWZseHpIax0oOOPW3cnwhOjRqbEJg7YRO/BDF7fO/PTWNT3T9Q== -mime-db@1.52.0: +mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== @@ -3091,6 +3116,11 @@ on-finished@2.4.1: dependencies: ee-first "1.1.1" +on-headers@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.2.tgz#772b0ae6aaa525c399e489adfad90c403eb3c28f" + integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA== + once@^1.3.0, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -3520,16 +3550,16 @@ safe-array-concat@^1.0.0, safe-array-concat@^1.0.1: has-symbols "^1.0.3" isarray "^2.0.5" +safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== - safe-json-stringify@~1: version "1.2.0" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz#356e44bc98f1f93ce45df14bcd7c01cda86e0afd"