New script to copy group associations from groups api to ubahn

Author: callmekatootie

update-groups-v2.js

@@ -0,0 +1,282 @@
+/**
+ * Purpose is same as update-groups.js => To copy groups info from groups
+ * api to ubahn. However, approach is different:
+ * - Read all users in ubahn
+ * - Get their topcoder user ids
+ * - Use the user ids to get the group associations in groups api
+ * - Copy the associations into ubahn
+ * 
+ * Run the script with `node update-groups-v2.js`
+ */
+const _ = require('lodash')
+const config = require('config')
+const axios = require('axios')
+const m2mAuth = require('tc-core-library-js').auth.m2m
+const elasticsearch = require('elasticsearch')
+
+const ubahnM2MConfig = _.pick(config, ['AUTH0_URL', 'AUTH0_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])
+const topcoderM2MConfig = _.pick(config, ['AUTH0_URL', 'AUTH0_TOPCODER_AUDIENCE', 'TOKEN_CACHE_TIME', 'AUTH0_PROXY_SERVER_URL'])
+
+const ubahnM2M = m2mAuth({ ...ubahnM2MConfig, AUTH0_AUDIENCE: ubahnM2MConfig.AUTH0_AUDIENCE })
+
+const topcoderM2M = m2mAuth({ ...topcoderM2MConfig, AUTH0_AUDIENCE: topcoderM2MConfig.AUTH0_TOPCODER_AUDIENCE })
+
+let esClient
+
+async function sleep (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+/**
+ * Get M2M token.
+ * @returns {Promise<unknown>}
+ */
+async function getTCM2MToken () {
+  return topcoderM2M.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET)
+}
+
+/**
+ * Returns m2m token for use with ubahn's apis
+ */
+async function getUbahnM2Mtoken () {
+  return ubahnM2M.getMachineToken(config.AUTH0_CLIENT_ID, config.AUTH0_CLIENT_SECRET)
+}
+
+/**
+ * Returns all groups associated with the member
+ * @param {String} memberId The member id
+ */
+async function getGroupsOfUser (memberId) {
+  const membershipType = 'user'
+  const url = `${config.GROUPS_API_URL}`
+  const perPage = 12
+  let page = 1
+  const groups = []
+  let once = false
+
+  const token = await getTCM2MToken()
+
+  while (true) {
+    try {
+      const res = await axios.get(url, {
+        params: {
+          page,
+          perPage,
+          memberId,
+          membershipType
+        },
+        headers: {
+          Authorization: `Bearer ${token}`
+        }
+      })
+
+      if (!once) {
+        const total = res.headers['x-total']
+        console.log(`Discovered ${total} groups...`)
+
+        if (total > 0) {
+          console.log('Fetching all of them...')
+        }
+
+        once = true
+      }
+
+      if (res.data.length > 0) {
+        groups.push(...res.data)
+      }
+
+      if (res.data.length !== perPage) {
+        break
+      }
+
+      page += 1
+    } catch (error) {
+      console.log(`Error when fetching groups at page ${page} and per page ${perPage} for member ${memberId}`)
+      console.log(error)
+
+      throw error
+    }
+  }
+
+  return groups
+}
+
+/**
+ * Returns all users in ubahn
+ */
+async function getAllUbahnUsers () {
+  const perPage = 12
+  let page = 1
+  const users = []
+  let once = false
+
+  const token = await getUbahnM2Mtoken()
+
+  while (true) {
+    try {
+      const res = await axios.get(config.UBAHN_USERS_API_URL, {
+        headers: {
+          Authorization: `Bearer ${token}`
+        }
+      })
+
+      if (!once) {
+        const total = res.headers['x-total']
+        console.log(`Discovered ${total} user(s) in ubahn...`)
+
+        if (total > 0) {
+          console.log('Fetching all of them...')
+        }
+
+        once = true
+      }
+
+      if (res.data.length > 0) {
+        users.push(...res.data)
+      }
+
+      if (res.data.length !== perPage) {
+        break
+      }
+
+      page += 1
+    } catch (error) {
+      console.log(`Error when fetching users in ubahn at page ${page} and per page ${perPage}`)
+      console.log(error)
+
+      throw error
+    }
+  }
+
+  return users
+}
+
+/**
+ * Returns the external profile of the user
+ * @param {String} userId The ubahn user id
+ */
+async function getExternalProfile (userId) {
+  const url = `${config.UBAHN_USERS_API_URL}/${userId}/externalProfiles`
+  const token = await getUbahnM2Mtoken()
+  try {
+    const res = await axios.get(url, {
+      headers: {
+        Authorization: `Bearer ${token}`
+      }
+    })
+
+    return res.data
+  } catch (error) {
+    console.log(`Error fetching the external profile of user with id ${userId}`)
+    console.log(error)
+
+    throw error
+  }
+}
+
+/**
+ * Returns the Elasticsearch client
+ */
+async function getESClient () {
+  const host = config.ES.HOST
+  const apiVersion = config.ES.API_VERSION
+
+  if (esClient) {
+    return esClient
+  }
+
+  // AWS ES configuration is different from other providers
+  if (/.*amazonaws.*/.test(host)) {
+    try {
+      esClient = new elasticsearch.Client({
+        apiVersion,
+        host,
+        connectionClass: require('http-aws-es') // eslint-disable-line global-require
+      })
+    } catch (error) { console.log(error) }
+  } else {
+    esClient = new elasticsearch.Client({
+      apiVersion,
+      host
+    })
+  }
+
+  return esClient
+}
+
+/**
+ * Updates the groups in the user
+ * @param {String} userId The user id
+ * @param {Array} groups The array of groups
+ */
+async function updateGroupsForUser (userId, groups) {
+  const client = await getESClient()
+  let user = await client.get({ index: config.get('ES.USER_INDEX'), type: config.get('ES.USER_TYPE'), id: userId })
+  user = user._source
+
+  const propertyName = config.get('ES.USER_GROUP_PROPERTY_NAME')
+  if (!user[propertyName]) {
+    user[propertyName] = []
+  }
+
+  let groupsTotal = user[propertyName].concat(groups)
+
+  groupsTotal = _.uniqBy(groupsTotal, (g) => g.groupId)
+
+  user[propertyName] = groupsTotal
+
+  await client.update({
+    index: config.get('ES.USER_INDEX'),
+    type: config.get('ES.USER_TYPE'),
+    id: userId,
+    body: { doc: user },
+    refresh: 'wait_for'
+  })
+}
+
+/**
+ * Main function
+ */
+async function start () {
+  // Get all users in ubahn
+  const users = await getAllUbahnUsers()
+  const final = {}
+
+  for (let i = 0; i < users.length; i++) {
+    const userId = users[i].id
+
+    console.log(`Getting external profile of user with id ${userId}`)
+
+    // Get external profiles of the user (to get the tc member ids of the ubahn users)
+    let externalProfiles = await getExternalProfile(userId)
+
+    externalProfiles = _.uniqBy(externalProfiles, (e) => e.externalId)
+
+    // Get groups of the user
+    for (let j = 0; j < externalProfiles.length; j++) {
+      const memberId = externalProfiles[j].externalId
+      console.log(`Getting groups of user with id ${memberId}`)
+      let groups = await getGroupsOfUser(memberId)
+      groups = groups.map(g => ({ groupId: g.id, name: g.name }))
+
+      if (groups.length > 0) {
+        if (!final[userId]) {
+          final[userId] = groups
+        } else {
+          final[userId] = final[userId].concat(groups)
+        }
+      }
+    }
+  }
+
+  const keys = Object.keys(final)
+
+  for (let i = 0; i < keys.length; i++) {
+    await updateGroupsForUser(keys[i], final[keys[i]])
+    await sleep()
+  }
+
+  console.log('All groups copied over to ubahn as relevant')
+}
+
+start()