Update elasticsearch host to support elastic cloud since we are using enrich feature, which is unsupported by AWS ES

Author: callmekatootie

README.md

@@ -27,7 +27,7 @@ The following parameters can be set in config files or in env variables:
 - GROUPS_MEMBER_ADD_TOPIC: the add groups member Kafka message topic, default value is 'groups.notification.member.add'
 - GROUPS_MEMBER_DELETE_TOPIC: the delete groups member Kafka message topic, default value is 'groups.notification.member.delete'
 - GROUPS_MEMBERSHIP_TYPE: the groups membership type that should be processed, default value is 'user'
-- ES.HOST: Elasticsearch host, default value is 'localhost:9200'
+- ES_HOST: Elasticsearch host, default value is 'localhost:9200'
 - ES.AWS_REGION: The Amazon region to use when using AWS Elasticsearch service, default value is 'us-east-1'
 - ES.ACHIEVEMENT_PROVIDER_INDEX: Elasticsearch index name for achievement provider, default value is 'achievement_provider'
 - ES.ACHIEVEMENT_PROVIDER_TYPE: Elasticsearch index type for achievement provider, default value is '_doc'
@@ -61,7 +61,9 @@ The following parameters can be set in config files or in env variables:
 - ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy`
 - SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
 - ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy`
-
+- ELASTICCLOUD_ID: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
+- ELASTICCLOUD_USERNAME: The elastic cloud username for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
+- ELASTICCLOUD_PASSWORD: The elastic cloud password for basic authentication. Provide this only if your elasticsearch instance is hosted on elastic cloud
 
 There is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
 

config/default.js

@@ -24,6 +24,13 @@ module.exports = {
 
   ES: {
     HOST: process.env.ES_HOST || 'http://localhost:9200',
+
+    ELASTICCLOUD: {
+      id: process.env.ELASTICCLOUD_ID,
+      username: process.env.ELASTICCLOUD_USERNAME,
+      password: process.env.ELASTICCLOUD_PASSWORD
+    },
+
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES
     ACHIEVEMENT_PROVIDER_INDEX: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider',
     ACHIEVEMENT_PROVIDER_TYPE: process.env.ACHIEVEMENT_PROVIDER_TYPE || '_doc',

package-lock.json

@@ -29,7 +29,6 @@
     "@elastic/elasticsearch": "^7.9.1",
     "@hapi/joi": "^15.1.0",
     "async-mutex": "^0.2.4",
-    "aws-elasticsearch-connector": "^9.0.0",
     "aws-sdk": "^2.476.0",
     "bluebird": "^3.5.5",
     "config": "^3.1.0",

package.json

@@ -5,7 +5,6 @@
 const AWS = require('aws-sdk')
 const config = require('config')
 const elasticsearch = require('@elastic/elasticsearch')
-const createAwsElasticsearchConnector = require('aws-elasticsearch-connector')
 const _ = require('lodash')
 const Joi = require('@hapi/joi')
 const { Mutex } = require('async-mutex')
@@ -40,15 +39,19 @@ async function getESClient () {
     return esClient
   }
   const host = config.ES.HOST
+  const cloudId = config.ES.ELASTICCLOUD.id
 
-  // AWS ES configuration is different from other providers
-  if (/.*amazonaws.*/.test(host)) {
-    try {
-      esClient = new elasticsearch.Client({
-        ...createAwsElasticsearchConnector(AWS.config),
-        node: host
-      })
-    } catch (error) { console.log(error) }
+  if (cloudId) {
+    // Elastic Cloud configuration
+    esClient = new elasticsearch.Client({
+      cloud: {
+        id: cloudId
+      },
+      auth: {
+        username: config.ES.ELASTICCLOUD.username,
+        password: config.ES.ELASTICCLOUD.password
+      }
+    })
   } else {
     esClient = new elasticsearch.Client({
       node: host