Merge pull request #19 from topcoder-platform/enrich

Integrate Enrichment of data feature

Author: callmekatootie

README.md

@@ -29,7 +29,6 @@ The following parameters can be set in config files or in env variables:
 - GROUPS_MEMBERSHIP_TYPE: the groups membership type that should be processed, default value is 'user'
 - ES.HOST: Elasticsearch host, default value is 'localhost:9200'
 - ES.AWS_REGION: The Amazon region to use when using AWS Elasticsearch service, default value is 'us-east-1'
-- ES.API_VERSION: Elasticsearch API version, default value is '7.4'
 - ES.ACHIEVEMENT_PROVIDER_INDEX: Elasticsearch index name for achievement provider, default value is 'achievement_provider'
 - ES.ACHIEVEMENT_PROVIDER_TYPE: Elasticsearch index type for achievement provider, default value is '_doc'
 - ES.ATTRIBUTE_INDEX: Elasticsearch index name for attribute, default value is 'attribute'
@@ -53,6 +52,16 @@ The following parameters can be set in config files or in env variables:
 - ES.USER_SKILL_PROPERTY_NAME: the user property name of skill, default value is 'skills'
 - ES.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME: the org property name of org skill providers, default value is 'skillProviders'
 - ES.USER_GROUP_PROPERTY_NAME: the user property name of group, default value is 'groups'
+- ATTRIBUTE_GROUP_PIPELINE_ID: The pipeline id for enrichment with attribute group. Default is `attributegroup-pipeline`
+- SKILL_PROVIDER_PIPELINE_ID: The pipeline id for enrichment with skill provider. Default is `skillprovider-pipeline`
+- USER_PIPELINE_ID: The pipeline id for enrichment of user details. Default is `user-pipeline`
+- ATTRIBUTE_GROUP_ENRICH_POLICYNAME: The enrich policy for attribute group. Default is `attributegroup-policy`
+- SKILL_PROVIDER_ENRICH_POLICYNAME: The enrich policy for skill provider. Default is `skillprovider-policy`
+- ROLE_ENRICH_POLICYNAME: The enrich policy for role. Default is `role-policy`
+- ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME: The enrich policy for achievement provider. Default is `achievementprovider-policy`
+- SKILL_ENRICH_POLICYNAME: The enrich policy for skill. Default is `skill-policy`
+- ATTRIBUTE_ENRICH_POLICYNAME: The enrich policy for skill. Default is `attribute-policy`
+
 
 There is a `/health` endpoint that checks for the health of the app. This sets up an expressjs server and listens on the environment variable `PORT`. It's not part of the configuration file and needs to be passed as an environment variable
 
@@ -103,15 +112,10 @@ Configuration for the tests is at `config/test.js`, only add such new configurat
     ```
 
 4. Initialize Elasticsearch index
+    For this, refer to the [Ubahn API](https://github.com/topcoder-platform/u-bahn-api) repository. In this repository, you need to execute the following script (after following their deployment guide):
 
     ```bash
-    npm run init-es
-    ```
-
-    To delete and re-create the index:
-
-    ```bash
-    npm run init-es force
+    npm run insert-data
     ```
 
 5. Start the processor and health check dropin

config/default.js

@@ -23,9 +23,8 @@ module.exports = {
   GROUPS_MEMBERSHIP_TYPE: process.env.GROUPS_MEMBERSHIP_TYPE || 'user',
 
   ES: {
-    HOST: process.env.ES_HOST || 'localhost:9200',
+    HOST: process.env.ES_HOST || 'http://localhost:9200',
     AWS_REGION: process.env.AWS_REGION || 'us-east-1', // AWS Region to be used if we use AWS ES
-    API_VERSION: process.env.ES_API_VERSION || '7.4',
     ACHIEVEMENT_PROVIDER_INDEX: process.env.ACHIEVEMENT_PROVIDER_INDEX || 'achievement_provider',
     ACHIEVEMENT_PROVIDER_TYPE: process.env.ACHIEVEMENT_PROVIDER_TYPE || '_doc',
     ATTRIBUTE_INDEX: process.env.ATTRIBUTE_INDEX || 'attribute',
@@ -50,6 +49,35 @@ module.exports = {
     USER_SKILL_PROPERTY_NAME: process.env.USER_SKILL_PROPERTY_NAME || 'skills',
     USER_GROUP_PROPERTY_NAME: process.env.USER_GROUP_PROPERTY_NAME || 'groups',
 
-    ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders'
+    ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME: process.env.ORGANIZATION_SKILLPROVIDER_PROPERTY_NAME || 'skillProviders',
+
+    ENRICHMENT: {
+      attributegroup: {
+        enrichPolicyName: process.env.ATTRIBUTE_GROUP_ENRICH_POLICYNAME || 'attributegroup-policy',
+        pipelineId: process.env.ATTRIBUTE_GROUP_PIPELINE_ID || 'attributegroup-pipeline'
+      },
+      skillprovider: {
+        enrichPolicyName: process.env.SKILL_PROVIDER_ENRICH_POLICYNAME || 'skillprovider-policy',
+        pipelineId: process.env.SKILL_PROVIDER_PIPELINE_ID || 'skillprovider-pipeline'
+      },
+      user: {
+        pipelineId: process.env.USER_PIPELINE_ID || 'user-pipeline'
+      },
+      role: {
+        enrichPolicyName: process.env.ROLE_ENRICH_POLICYNAME || 'role-policy'
+      },
+      achievementprovider: {
+        enrichPolicyName: process.env.ACHIEVEMENT_PROVIDER_ENRICH_POLICYNAME || 'achievementprovider-policy'
+      },
+      skill: {
+        enrichPolicyName: process.env.SKILL_ENRICH_POLICYNAME || 'skill-policy'
+      },
+      attribute: {
+        enrichPolicyName: process.env.ATTRIBUTE_ENRICH_POLICYNAME || 'attribute-policy'
+      },
+      organization: {
+        enrichPolicyName: process.env.ORGANIZATION_ENRICH_POLICYNAME || 'organization-policy'
+      }
+    }
   }
 }

docker-kafka-es/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       KAFKA_CREATE_TOPICS: "u-bahn.action.aggregate:1:1,groups.notification.member.add:1:1,groups.notification.member.delete:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   esearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2
+    image: elasticsearch:7.7.1
     container_name: ubahn-data-processor-es_es
     ports:
       - "9200:9200"

package-lock.json

@@ -26,14 +26,14 @@
     "superagent": "^5.1.0"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^7.9.1",
     "@hapi/joi": "^15.1.0",
     "async-mutex": "^0.2.4",
+    "aws-elasticsearch-connector": "^9.0.0",
     "aws-sdk": "^2.476.0",
     "bluebird": "^3.5.5",
     "config": "^3.1.0",
-    "elasticsearch": "^16.7.1",
     "get-parameter-names": "^0.3.0",
-    "http-aws-es": "^6.0.0",
     "lodash": "^4.17.19",
     "no-kafka": "^3.4.3",
     "topcoder-healthcheck-dropin": "^1.0.3",