Create db migrations

README.md

@@ -67,7 +67,7 @@ Setup your Elasticsearch instance and ensure that it is up and running.
 
 ## Working with mock data
 
-You can use the scripts `npm run insert-data` (and `npm run delete-data`) to insert mock data (and delete mock data respectively). The data is inserted into Postgres and Elasticsearch. You need to setup the configurations beforehand and also start the elasticsearch instance before you run these scripts. You can run the script `npm run migrate-db-to-es` to dump data in db into es.
+You can use the scripts `npm run migrations up` (and `npm run migrations down`) to insert mock data (and delete mock data respectively). The data is inserted into Postgres and Elasticsearch. You need to setup the configurations beforehand and also start the elasticsearch instance before you run these scripts. You can run the script `npm run migrate-db-to-es` to dump data in db into es.
 
 ## Local Deployment with Docker
 

package.json

@@ -8,7 +8,8 @@
     "lint": "standard \"**/*.js\"",
     "insert-data": "node scripts/db/genData.js",
     "delete-data": "node scripts/db/dropAll.js",
-    "migrate-db-to-es": "node scripts/db/dumpDbToEs.js"
+    "migrate-db-to-es": "node scripts/db/dumpDbToEs.js",
+    "migrations": "node scripts/db/migrations.js"
   },
   "repository": {
     "type": "git",
@@ -36,6 +37,7 @@
     "swagger-ui-express": "^4.1.4",
     "tc-bus-api-wrapper": "github:topcoder-platform/tc-bus-api-wrapper",
     "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6.4",
+    "umzug": "^2.3.0",
     "uuid": "^7.0.1",
     "winston": "^3.2.1"
   },

scripts/db/data/ExternalProfile.json

@@ -24,7 +24,7 @@
     "uri": "http://www.new.com/new-uri"
   },
   {
-    "id": "f2d1b567-8ea3-4eec-93b0-32378a19edb7",
+    "id": "f2d1b567-8eb3-4eec-93b0-32378a19edb7",
     "created": "2020-05-13T06:11:21.361Z",
     "updated": "2020-05-13T06:46:15.893Z",
     "createdBy": "tc-Admin",
@@ -36,7 +36,7 @@
     "uri": "http://www.new.com/new-uri"
   },
   {
-    "id": "f2d1b567-8ea3-4eec-93b0-32378a19edb7",
+    "id": "f2d1b567-8ec3-4eec-93b0-32378a19edb7",
     "created": "2020-05-13T06:11:21.361Z",
     "updated": "2020-05-13T06:46:15.893Z",
     "createdBy": "tc-Admin",

scripts/db/genData.js

@@ -2,12 +2,10 @@
 
 const _ = require('lodash')
 const sequelize = require('../../src/models/index')
-const dbHelper = require('../../src/common/db-helper')
 const logger = require('../../src/common/logger')
 const { getESClient } = require('../../src/common/es-client')
 const {
   topResources,
-  userResources,
   organizationResources,
   modelToESIndexMapping
 } = require('../constants')
@@ -32,98 +30,6 @@ const RESOURCES_IN_ORDER = [
 
 const client = getESClient()
 
-async function insertIntoES (modelName, body) {
-  const esResourceName = modelToESIndexMapping[modelName]
-
-  if (!esResourceName) {
-    logger.error(`Cannot insert data into model ${modelName}. No equivalent elasticsearch index found`)
-
-    return
-  }
-
-  if (_.includes(_.keys(topResources), esResourceName)) {
-    await client.index({
-      index: topResources[esResourceName].index,
-      type: topResources[esResourceName].type,
-      id: body.id,
-      body,
-      pipeline: topResources[esResourceName].ingest ? topResources[esResourceName].ingest.pipeline.id : undefined,
-      refresh: 'wait_for'
-    })
-  } else if (_.includes(_.keys(userResources), esResourceName)) {
-    const userResource = userResources[esResourceName]
-
-    const { body: user } = await client.getSource({
-      index: topResources.user.index,
-      type: topResources.user.type,
-      id: body.userId
-    })
-
-    if (userResource.nested === true && userResource.mappingCreated !== true) {
-      await client.indices.putMapping({
-        index: topResources.user.index,
-        type: topResources.user.type,
-        include_type_name: true,
-        body: {
-          properties: {
-            [userResource.propertyName]: {
-              type: 'nested'
-            }
-          }
-        }
-      })
-      userResource.mappingCreated = true
-    }
-
-    const relateId = body[userResource.relateKey]
-
-    if (!user[userResource.propertyName]) {
-      user[userResource.propertyName] = []
-    }
-
-    if (_.some(user[userResource.propertyName], [userResource.relateKey, relateId])) {
-      logger.error(`Can't create existing ${esResourceName} with the ${userResource.relateKey}: ${relateId}, userId: ${body.userId}`)
-    } else {
-      user[userResource.propertyName].push(body)
-      await client.index({
-        index: topResources.user.index,
-        type: topResources.user.type,
-        id: body.userId,
-        body: user,
-        pipeline: topResources.user.pipeline.id,
-        refresh: 'wait_for'
-      })
-    }
-  } else if (_.includes(_.keys(organizationResources), esResourceName)) {
-    const orgResource = organizationResources[esResourceName]
-
-    const { body: organization } = await client.getSource({
-      index: topResources.organization.index,
-      type: topResources.organization.type,
-      id: body.organizationId
-    })
-
-    const relateId = body[orgResource.relateKey]
-
-    if (!organization[orgResource.propertyName]) {
-      organization[orgResource.propertyName] = []
-    }
-
-    if (_.some(organization[orgResource.propertyName], [orgResource.relateKey, relateId])) {
-      logger.error(`Can't create existing ${esResourceName} with the ${orgResource.relateKey}: ${relateId}, organizationId: ${body.organizationId}`)
-    } else {
-      organization[orgResource.propertyName].push(body)
-      await client.index({
-        index: topResources.organization.index,
-        type: topResources.organization.type,
-        id: body.organizationId,
-        body: organization,
-        refresh: 'wait_for'
-      })
-    }
-  }
-}
-
 /**
  * Creates and executes the enrich policy for the provided model
  * @param {String} modelName The model name
@@ -219,10 +125,6 @@ async function createEnrichProcessor (modelName) {
  * @return {Promise<void>}
  */
 async function main () {
-  await dbHelper.createDb()
-  await sequelize.sync()
-  dbHelper.addRelationships(sequelize)
-
   let keys = Object.keys(sequelize.models)
 
   // Sort the models in the order of insertion (for correct enrichment)
@@ -238,20 +140,6 @@ async function main () {
 
   for (let i = 0; i < keys.length; i++) {
     const key = keys[i]
-    try {
-      const data = require(`./data/${key}.json`)
-      for (let i = 0; i < data.length; i++) {
-        logger.info(`Inserting data ${i + 1} of ${data.length}`)
-        await dbHelper.create(sequelize.models[key], data[i], null)
-        // await insertIntoES(key, data[i])
-      }
-      logger.info('import data for ' + key + ' done')
-    } catch (e) {
-      logger.error(e)
-      logger.warn('import data for ' + key + ' failed')
-      continue
-    }
-
     try {
       await createAndExecuteEnrichPolicy(key)
       logger.info('create and execute enrich policy for ' + key + ' done')

scripts/db/migrations.js

@@ -0,0 +1,35 @@
+const sequelize = require('../../src/models/index')
+const path = require('path')
+const Umzug = require('umzug')
+const { createDb } = require('../../src/common/db-helper')
+
+const umzug = new Umzug({
+  migrations: {
+    // indicates the folder containing the migration .js files
+    path: path.join(__dirname, './migrations'),
+    // inject sequelize's QueryInterface in the migrations
+    params: [
+      sequelize.getQueryInterface()
+    ]
+  },
+  // indicates that the migration data should be store in the database
+  // itself through sequelize. The default configuration creates a table
+  // named `SequelizeMeta`.
+  storage: 'sequelize',
+  storageOptions: {
+    sequelize: sequelize
+  }
+});
+
+(async () => {
+  if (process.argv[2] === 'up') {
+    createDb()
+    await umzug.up()
+    console.log('All migrations performed successfully')
+  } else if (process.argv[2] === 'down') {
+    await umzug.down()
+    console.log('The last executed migration reverted successfully')
+  } else {
+    console.error('You should pass \'up\' or \'down\' as the parameter')
+  }
+})()

scripts/db/migrations/00_create-achievement.js

@@ -0,0 +1,44 @@
+/**
+ * Achievement model
+ */
+const { DataTypes } = require('sequelize')
+
+module.exports = {
+  up: async (query) => {
+    await query.createTable('Achievements', {
+      id: {
+        primaryKey: true,
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4
+      },
+      createdBy: {
+        type: DataTypes.STRING
+      },
+      updatedBy: {
+        type: DataTypes.STRING
+      },
+      name: {
+        type: DataTypes.STRING
+      },
+      uri: {
+        type: DataTypes.STRING
+      },
+      certifierId: {
+        type: DataTypes.STRING
+      },
+      certifiedDate: {
+        type: DataTypes.DATE
+      },
+      created: {
+        type: DataTypes.DATE,
+        allowNull: false
+      },
+      updated: {
+        type: DataTypes.DATE
+      }
+    })
+  },
+  down: async (query) => {
+    await query.dropTable('Achievements')
+  }
+}

scripts/db/migrations/01_create-achievementsProvider.js

@@ -0,0 +1,35 @@
+/**
+ * AchievementsProvider model
+ */
+const { DataTypes } = require('sequelize')
+
+module.exports = {
+  up: async (query) => {
+    await query.createTable('AchievementsProviders', {
+      id: {
+        primaryKey: true,
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4
+      },
+      createdBy: {
+        type: DataTypes.STRING
+      },
+      updatedBy: {
+        type: DataTypes.STRING
+      },
+      name: {
+        type: DataTypes.STRING
+      },
+      created: {
+        type: DataTypes.DATE,
+        allowNull: false
+      },
+      updated: {
+        type: DataTypes.DATE
+      }
+    })
+  },
+  down: async (query) => {
+    await query.dropTable('AchievementsProviders')
+  }
+}

scripts/db/migrations/02_create-attribute.js

@@ -0,0 +1,35 @@
+/**
+ * Attribute model
+ */
+const { DataTypes } = require('sequelize')
+
+module.exports = {
+  up: async (query) => {
+    await query.createTable('Attributes', {
+      id: {
+        primaryKey: true,
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4
+      },
+      createdBy: {
+        type: DataTypes.STRING
+      },
+      updatedBy: {
+        type: DataTypes.STRING
+      },
+      name: {
+        type: DataTypes.STRING
+      },
+      created: {
+        type: DataTypes.DATE,
+        allowNull: false
+      },
+      updated: {
+        type: DataTypes.DATE
+      }
+    })
+  },
+  down: async (query) => {
+    await query.dropTable('Attributes')
+  }
+}

scripts/db/migrations/03_create-attributeGroup.js

@@ -0,0 +1,35 @@
+/**
+ * AttributeGroup model
+ */
+const { DataTypes } = require('sequelize')
+
+module.exports = {
+  up: async (query) => {
+    await query.createTable('AttributeGroups', {
+      id: {
+        primaryKey: true,
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4
+      },
+      createdBy: {
+        type: DataTypes.STRING
+      },
+      updatedBy: {
+        type: DataTypes.STRING
+      },
+      name: {
+        type: DataTypes.STRING
+      },
+      created: {
+        type: DataTypes.DATE,
+        allowNull: false
+      },
+      updated: {
+        type: DataTypes.DATE
+      }
+    })
+  },
+  down: async (query) => {
+    await query.dropTable('AttributeGroups')
+  }
+}