Merge pull request #632 from topcoder-platform/dev

[PROD] Next Release

Author: sachin-maheshwari

.vscode/launch.json

@@ -0,0 +1,40 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "name": "Nodemon TaaS API",
+      "program": "${workspaceFolder}/app.js",
+      "envFile": "${workspaceFolder}/.env",
+      "request": "launch",
+      "restart": true,
+      "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/nodemon",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "type": "pwa-node"
+    },
+    {
+      "type": "pwa-node",
+      "request": "launch",
+      "name": "Launch TaaS API",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "program": "${workspaceFolder}/app.js",
+      "envFile": "${workspaceFolder}/.env",
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Attach to dev:debug",
+      "type": "node",
+      "request": "attach",
+      "restart": true,
+      "port": 9229
+    }
+  ]
+}

Makefile

@@ -0,0 +1,43 @@
+#!/bin/bash
+GR=\033[92m>>>
+NC=  <<<\033[0m
+.PHONY: dump_tables
+dump_tables:
+	@echo "${GR}make sure you checked in all your changes${NC}"
+	@echo "${GR}going to dev${NC}"
+	git checkout dev
+	npm run services:down
+	npm run services:up
+	npm run init-db
+	npm run migrate
+	@echo "${GR}cool, we are now migrated to dev status... moving on to our branch${NC}"
+	git checkout feature/interview-nylas
+	npm run migrate
+	@echo "${GR}now we are post-feature migration state, let's dump the tables${NC}"
+	@mkdir -p ./.comparisons/migrate
+	@mkdir -p ./.comparisons/init-db
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.interviews' --schema-only postgres > ./.comparisons/migrate/interviews.sql
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.job_candidates' --schema-only postgres > ./.comparisons/migrate/job_candidates.sql
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.user_meeting_settings' --schema-only postgres > ./.comparisons/migrate/user_meeting_settings.sql
+	@echo "${GR}now we revert and simply `init-db force` from the feature branch${NC}"
+	npm run services:down
+	npm run services:up
+	npm run init-db force
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.interviews' --schema-only postgres > ./.comparisons/init-db/interviews.sql
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.job_candidates' --schema-only postgres > ./.comparisons/init-db/job_candidates.sql
+	@docker exec -t tc-taas-postgres pg_dump -h localhost --username=postgres -t 'bookings.user_meeting_settings' --schema-only postgres > ./.comparisons/init-db/user_meeting_settings.sql
+	@echo "${GR}All done, you can now compare the files${NC}"
+	git diff --no-index ./.comparisons/migrate/interviews.sql ./.comparisons/init-db/interviews.sql > ./.comparisons/interviews.diff || true
+	git diff --no-index ./.comparisons/migrate/job_candidates.sql ./.comparisons/init-db/job_candidates.sql > ./.comparisons/job_candidates.diff || true
+	git diff --no-index ./.comparisons/migrate/user_meeting_settings.sql ./.comparisons/init-db/user_meeting_settings.sql > ./.comparisons/user_meeting_settings.diff || true
+
+
+
+.PHONY: reboot
+reboot:
+	npm run services:down
+	npm run services:up
+	npm run init-db
+	npm run migrate
+	npm run local:init || true
+	npm run test

README.md

@@ -40,13 +40,15 @@
       AUTH0_AUDIENCE_UBAHN=
       AUTH0_CLIENT_ID=
       AUTH0_CLIENT_SECRET=
-      # necessary if you'll utilize email functionality of interviews
-      INTERVIEW_INVITATION_SENDGRID_TEMPLATE_ID=
-      INTERVIEW_INVITATION_SENDER_EMAIL=
+      # If you would like to test Interview Workflow then Config Nylas as per ./docs/guides/Setup-Interview-Workflow-Locally.md
+      NYLAS_CLIENT_ID=
+      NYLAS_CLIENT_SECRET=
+      NYLAS_SCHEDULER_WEBHOOK_BASE_URL=
       # Locally deployed services (via docker-compose)
       ES_HOST=http://dockerhost:9200
       DATABASE_URL=postgres://postgres:postgres@dockerhost:5432/postgres
       BUSAPI_URL=http://dockerhost:8002/v5
+      TAAS_API_BASE_URL=http://localhost:3000/api/v5
       # stripe
       STRIPE_SECRET_KEY=
       CURRENCY=usd
@@ -55,7 +57,7 @@
       - Values from this file would be automatically used by many `npm` commands.
       - ⚠️ Never commit this file or its copy to the repository!
 
-   1. Set `dockerhost` to point the IP address of Docker. Docker IP address depends on your system. For example if docker is run on IP `127.0.0.1` add a the next line to your `/etc/hosts` file:
+   2. Set `dockerhost` to point the IP address of Docker. Docker IP address depends on your system. For example if docker is run on IP `127.0.0.1` add a the next line to your `/etc/hosts` file:
 
       ```
       127.0.0.1       dockerhost
@@ -170,6 +172,8 @@
    Runs the Topcoder TaaS API using nodemon, so it would be restarted after any of the files is updated.
    The Topcoder TaaS API will be served on `http://localhost:3000`.
 
+   - 💡 If you would like to test Interview Workflow locally, then follow the guide [How to Setup Interview Workflow Locally](./docs/guides/Setup-Interview-Workflow-Locally.md).
+
 ### Working on `taas-es-processor` locally
 
 When you run `taas-apis` locally as per "[Steps to run locally](#steps-to-run-locally)" the [taas-es-processor](https://github.com/topcoder-platform/taas-es-processor) would be run for you automatically together with other services inside the docker container via `npm run services:up`.
@@ -224,7 +228,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run cov`                                                                                                             | Code Coverage Report.                                                                                                                              |
 | `npm run migrate`                                                                                                         | Run any migration files which haven't run yet.                                                                                                     |
 | `npm run migrate:undo`                                                                                                    | Revert most recent migration.                                                                                                                      |
-| `npm run demo-payment-scheduler`                                                                                          | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount"                                                           |
+| `npm run demo-email-notifications`                                                                                          | Listen to the Kafka events of email notification and render all the emails into `./out` folder. See [its readme](scripts/demo-email-notifications/README.md) for details. |
 | `npm run emsi-mapping`                                                                                                    | mapping EMSI tags to topcoder skills                                                                                                               |
 
 ## Import and Export data
@@ -294,6 +298,29 @@ docker exec -it tc-taas-kafka /opt/kafka/bin/kafka-console-producer.sh --broker-
 
 - Enter or copy/paste the message into the console after starting this command.
 
+## Email Notifications
+
+We have various email notifications. For example many emails are sent to support Interview Scheduling Workflow. All email templates are placed inside the [.data/notification-email-templates](./data/notification-email-templates/) folder.
+
+### Add a new Email Notification
+
+To add a new email notification:
+
+0. Each email notification need to have a unique topic identifier of the shape `taas.notification.{notification-type}`. Where `{notification-type}` is unique for each email notification type.
+1. Create a new HTML template inside folder  [.data/notification-email-templates](./data/notification-email-templates/). (You may duplicate any existent template to reuse existent styles.). Name it the same as topic: `taas.notification.{notification-type}.html`.
+2. Create a corresponding config in file [./config/email_template.config.js](./config/email_template.config.js), section `notificationEmailTemplates`.
+3. Name environment variable the same as topic, but **uppercase** and replace all special symbols to `_` and add suffix `_SENDGRID_TEMPLATE_ID`:
+   - For example topic `taas.notification.job-candidate-resume-viewed` would have corresponding environment variable
+`TAAS_NOTIFICATION_JOB_CANDIDATE_RESUME_VIEWED_SENDGRID_TEMPLATE_ID`.
+4. When deploying to DEV/PROD someone would have to create a new Sendgird template and fill inside Sendgrid UI subject and email HTML template by copy/pasting HTML file from the repo. And then set environment variable with the value of template if provided by Sendgrid.
+
+### Test/Render Email Notifications Locally
+
+To test and render email notification locally run a special script `npm run demo-email-notifications`. Before running it, follow it's [README](./scripts/demo-email-notifications/README.md) as you would have to set some additional environment variables first (add them into `.env` file).
+
+- This script first would update demo data to create some situations to trigger notifications.
+- And it would listen to Kafka events and render email notification into `./out` folder.
+
 ## DB Migration
 
 - `npm run migrate`: run any migration files which haven't run yet.
@@ -303,10 +330,7 @@ Configuration for migration is at `./config/config.json`.
 
 The following parameters can be set in the config file or via env variables:
 
-- `username`: set via env `DB_USERNAME`; datebase username
-- `password`: set via env `DB_PASSWORD`; datebase password
-- `database`: set via env `DB_NAME`; datebase name
-- `host`: set via env `DB_HOST`; datebase host name
+- `url`: set via env `DATABASE_URL`; datebase url
 
 ## Testing
 
@@ -341,6 +365,7 @@ When we add, update or delete models and/or endpoints we have to make sure that
   - Test, that when we migrate DB from the previous state using `npm run migrate`, we get exactly the same DB schema as if we create DB from scratch using command `npm run init-db force`.
 
 ## EMSI mapping
+
 mapping EMSI tags to topcoder skills
 Run `npm run emsi-mapping` to create the mapping file
 It will take about 15 minutes to create the mapping file `script/emsi-mapping/emsi-skils-mapping.js`

app-constants.js

@@ -58,7 +58,13 @@ const Scopes = {
   CREATE_ROLE: 'create:taas-roles',
   UPDATE_ROLE: 'update:taas-roles',
   DELETE_ROLE: 'delete:taas-roles',
-  ALL_ROLE: 'all:taas-roles'
+  ALL_ROLE: 'all:taas-roles',
+  // userMeetingSettings
+  READ_USER_MEETING_SETTINGS: 'read:taas-userMeetingsSettings',
+  CREATE_USER_MEETING_SETTINGS: 'create:taas-userMeetingsSettings',
+  UPDATE_USER_MEETING_SETTINGS: 'update:taas-userMeetingsSettings',
+  ALL_USER_MEETING_SETTINGS: 'all:taas-userMeetingsSettings'
+
 }
 
 // Interview related constants
@@ -69,14 +75,22 @@ const Interviews = {
     RequestedForReschedule: 'Requested for reschedule',
     Rescheduled: 'Rescheduled',
     Completed: 'Completed',
-    Cancelled: 'Cancelled'
-  },
-  // key: template name in x.ai, value: duration
-  XaiTemplate: {
-    'interview-30': 30,
-    'interview-60': 60
+    Cancelled: 'Cancelled',
+    Expired: 'Expired'
   },
-  MaxAllowedCount: 3
+  MaxAllowedCount: 3,
+  Nylas: {
+    Days: {
+      Monday: 'M',
+      Tuesday: 'T',
+      Wednesday: 'W',
+      Thursday: 'R',
+      Friday: 'F',
+      Saturday: 'S',
+      Sunday: 'U'
+    },
+    StartEndRegex: /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]$/
+  }
 }
 
 const ChallengeStatus = {
@@ -166,7 +180,23 @@ const JobCandidateStatus = {
   INTERVIEW: 'interview'
 }
 
+const SearchUsers = {
+  SEARCH_USERS_PAGE_SIZE: 5
+}
+
+// provider which we have to enforce for Nylas Virtual Calendars
+const NylasVirtualCalendarProvider = 'nylas'
+
+const ZoomLinkType = {
+  HOST: 'host',
+  GUEST: 'guest'
+}
+
+// how long to wait for the Interview Webhook Mutes to release (ms)
+const InterviewEventHandlerTimeout = 60 * 1000 // 60 seconds
+
 module.exports = {
+  InterviewEventHandlerTimeout,
   UserRoles,
   FullManagePermissionRoles,
   Scopes,
@@ -181,5 +211,8 @@ module.exports = {
   WeeklySurveySwitch,
   ActiveWorkPeriodPaymentStatuses,
   JobStatus,
-  JobCandidateStatus
+  JobCandidateStatus,
+  SearchUsers,
+  NylasVirtualCalendarProvider,
+  ZoomLinkType
 }

app.js

@@ -25,7 +25,32 @@ app.use(cors({
   // Allow browsers access pagination data in headers
   exposedHeaders: ['X-Page', 'X-Per-Page', 'X-Total', 'X-Total-Pages', 'X-Prev-Page', 'X-Next-Page']
 }))
-app.use(express.json())
+app.use((...args) => {
+  const [req, res, next] = args
+  // For test nylas webhook, we need raw buffer
+  // Here i sCustom Middleware to compute rawBody. Unfortunately using
+  // JSON.stringify(req.body) will remove spaces and newlines, so verification
+  // will fail. We must add this middleware to ensure we're computing the correct
+  // signature
+  if (req.path.match(/\/taas-teams\/nylas-webhooks/i)) {
+    req.rawBody = ''
+    req.on('data', (chunk) => (req.rawBody += chunk))
+    req.on('error', () => res.status(500).send('Error parsing body'))
+
+    req.on('end', () => {
+      // because the stream has been consumed, other parsers like bodyParser.json
+      // cannot stream the request data and will time out so we must explicitly parse the body
+      try {
+        req.body = req.rawBody.length ? JSON.parse(req.rawBody) : {}
+        next()
+      } catch (err) {
+        res.status(500).send('Error parsing body')
+      }
+    })
+    return
+  }
+  return express.json()(...args)
+})
 app.use(express.urlencoded({ extended: true }))
 app.set('port', config.PORT)
 
@@ -100,7 +125,7 @@ const server = app.listen(app.get('port'), () => {
   logger.info({ component: 'app', message: `Express server listening on port ${app.get('port')}` })
   eventHandlers.init()
   // schedule updateCompletedInterviews to run every hour
-  schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
+  schedule.scheduleJob(config.CRON_UPDATE_COMPLETED_INTERVIEWS, interviewService.updateCompletedInterviews)
   // schedule sendSurveys
   if (WeeklySurveySwitch.ON === config.WEEKLY_SURVEY.SWITCH) {
     schedule.scheduleJob(config.WEEKLY_SURVEY.CRON, sendSurveys)
@@ -113,6 +138,9 @@ const server = app.listen(app.get('port'), () => {
   schedule.scheduleJob(config.CRON_INTERVIEW_COMPLETED, notificationSchedulerService.sendInterviewCompletedNotifications)
   schedule.scheduleJob(config.CRON_POST_INTERVIEW, notificationSchedulerService.sendPostInterviewActionNotifications)
   schedule.scheduleJob(config.CRON_UPCOMING_RESOURCE_BOOKING, notificationSchedulerService.sendResourceBookingExpirationNotifications)
+
+  schedule.scheduleJob(config.CRON_INTERVIEW_EXPIRED, notificationSchedulerService.sendInterviewExpiredNotifications)
+  schedule.scheduleJob(config.CRON_INTERVIEW_SCHEDULE_REMINDER, notificationSchedulerService.sendInterviewScheduleReminderNotifications)
 })
 
 if (process.env.NODE_ENV === 'test') {

config/config.js

@@ -4,24 +4,12 @@
 
 module.exports = {
   development: {
-    username: process.env.DB_USERNAME || 'postgres',
-    password: process.env.DB_PASSWORD || 'postgres',
-    database: process.env.DB_NAME || 'postgres',
-    host: process.env.DB_HOST || '127.0.0.1',
-    dialect: 'postgres'
+    url: process.env.DATABASE_URL || 'postgres://postgres:postgres@localhost:5432/postgres'
   },
   test: {
-    username: process.env.DB_USERNAME || 'postgres',
-    password: process.env.DB_PASSWORD || 'postgres',
-    database: process.env.DB_NAME || 'postgres',
-    host: process.env.DB_HOST || '127.0.0.1',
-    dialect: 'postgres'
+    url: process.env.DATABASE_URL || 'postgres://postgres:postgres@localhost:5432/postgres'
   },
   production: {
-    username: process.env.DB_USERNAME,
-    password: process.env.DB_PASSWORD,
-    database: process.env.DB_NAME,
-    host: process.env.DB_HOST,
-    dialect: 'postgres'
+    url: process.env.DATABASE_URL
   }
 }