Initial commit

Author: jmgasper

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

.eslintrc.js

@@ -0,0 +1,13 @@
+module.exports = {
+  extends: 'standard-with-typescript',
+  parserOptions: {
+    project: './tsconfig.json'
+  },
+  rules: {
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/strict-boolean-expressions': 'off',
+    '@typescript-eslint/no-non-null-assertion': 'off',
+    '@typescript-eslint/no-dynamic-delete': 'off',
+    '@typescript-eslint/dot-notation': 'off'
+  }
+}

.gitignore

@@ -0,0 +1,98 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TypeScript output
+dist
+out
+
+# Azure Functions artifacts
+bin
+obj
+appsettings.json
+local.settings.json
+
+.idea
+.build
+.DS_Store

.nvmrc

@@ -0,0 +1 @@
+v12.20.1

DLPTrigger/handlers/get.ts

@@ -0,0 +1,21 @@
+import Boom from '@hapi/boom'
+import * as aws from 'aws-lambda'
+
+import { getItem } from '../models/AdoWorkItemsDlpStatus'
+
+/**
+ * Handles HTTP get event.
+ * @param event
+ * @param context
+ */
+export default async function handleGetRequest (event: aws.APIGatewayEvent, context: aws.Context) {
+  const projectId = event.queryStringParameters?.project_id
+  if (!projectId) {
+    throw Boom.badRequest('project_id is required')
+  }
+  const resourceId = event.queryStringParameters?.resource_id
+  if (!resourceId) {
+    throw Boom.badRequest('resource_id is required')
+  }
+  return await getItem(projectId, resourceId)
+}

DLPTrigger/handlers/post.ts

@@ -0,0 +1,150 @@
+import * as aws from 'aws-lambda'
+import Boom from '@hapi/boom'
+import _ from 'lodash'
+import { htmlToText } from 'html-to-text'
+
+import { identifyPII } from '../utils/presidio'
+import { DlpStatus, getItem, setNoErrors, StatusFields } from '../models/AdoWorkItemsDlpStatus'
+
+import {
+  FieldMapItem,
+  PROJECT_ID_FIELD_PATH,
+  WORKITEM_TYPE_FIELD_PATHS,
+  RESOURCE_ID_FIELD_PATHS,
+  TARGET_FIELDS,
+  WORKITEM_TYPES
+} from '../utils/field-path-map'
+
+/**
+ * Field map record interface.
+ */
+interface FieldMapRecord {
+  startIndex: number | null
+  endIndex: number | null
+  fieldMapItem: FieldMapItem
+  found: boolean
+  isString: boolean
+  rawValue: string | null
+  processedValue: string | null
+}
+
+/**
+ * Handles HTTP POST request.
+ * @param event AWS lambda event
+ * @param _context AWS lambda context
+ */
+export default async function handlePostRequest (event: aws.APIGatewayEvent, _context: aws.Context) {
+  if (_.isNull(event.body)) {
+    throw Boom.badRequest('Request does not have a body.')
+  }
+  const body = JSON.parse(event.body)
+  let workItemType: WORKITEM_TYPES | undefined
+  for (const workItemTypeFieldPath of WORKITEM_TYPE_FIELD_PATHS) {
+    workItemType = _.get(body, workItemTypeFieldPath)
+    if (workItemType) {
+      break
+    }
+  }
+  if (!workItemType) {
+    throw Boom.badRequest(`Did not find expected property at path: ${WORKITEM_TYPE_FIELD_PATHS.map(i => i.join('.')).join(', ')}`)
+  }
+  const projectId = _.get(body, PROJECT_ID_FIELD_PATH)
+  if (!projectId) {
+    throw Boom.badRequest(`Did not find expected property at path: ${PROJECT_ID_FIELD_PATH.join(', ')}`)
+  }
+  let resourceId: string | undefined
+  for (const resourceIdFieldPath of RESOURCE_ID_FIELD_PATHS) {
+    resourceId = _.get(body, resourceIdFieldPath)
+    if (resourceId) {
+      break
+    }
+  }
+  if (!resourceId) {
+    throw Boom.badRequest(`Did not find expected property at path: ${RESOURCE_ID_FIELD_PATHS.map(i => i.join('.')).join(', ')}`)
+  }
+  const fieldMap = TARGET_FIELDS[workItemType]
+  if (!fieldMap) {
+    throw Boom.badRequest(`Unexpected resource of type ${workItemType}. Accepted values: ${Object.keys(TARGET_FIELDS).join(', ')}.`)
+  }
+  const records: FieldMapRecord[] = []
+  let recordString: string = ''
+  for (const fieldItem of fieldMap) {
+    let fieldValue: string | null = null
+    for (const fieldPath of fieldItem.fieldPaths) {
+      fieldValue = _.get(body, fieldPath)
+      if (fieldValue) {
+        break
+      }
+    }
+    if (!fieldValue || !_.isString(fieldValue)) {
+      records.push({
+        startIndex: null,
+        endIndex: null,
+        fieldMapItem: fieldItem,
+        found: !!fieldValue,
+        isString: _.isString(fieldValue),
+        rawValue: null,
+        processedValue: null
+      })
+      continue
+    }
+    const plainTextFieldValue: string = htmlToText(fieldValue)
+    records.push({
+      startIndex: recordString.length,
+      endIndex: recordString.length + plainTextFieldValue.length + 1,
+      fieldMapItem: fieldItem,
+      found: true,
+      isString: true,
+      rawValue: fieldValue,
+      processedValue: plainTextFieldValue
+    })
+    recordString = `${recordString}${plainTextFieldValue}\n`
+  }
+  const piiDetailList = await identifyPII(recordString)
+  const adoWorkItemsDlpStatus = await getItem(projectId, resourceId)!
+  if (!adoWorkItemsDlpStatus) {
+    throw Boom.internal('Failed to get work item dlp status')
+  }
+  await setNoErrors(adoWorkItemsDlpStatus)
+  if (_.isNil(piiDetailList)) {
+    for (const statusField of Object.values(StatusFields)) {
+      adoWorkItemsDlpStatus[statusField].status = DlpStatus.NO_ISSUES
+      adoWorkItemsDlpStatus[statusField].issues = []
+    }
+    return { message: 'OK' }
+  }
+  for (const piiDetail of piiDetailList) {
+    const startIdx = piiDetail.start
+    for (const recordItem of records) {
+      if (!recordItem.isString || !recordItem.found) {
+        continue
+      }
+      if (recordItem.startIndex! <= startIdx && startIdx < recordItem.endIndex!) {
+        const piiMatch = recordString.substr(piiDetail.start, piiDetail.end - piiDetail.start)
+        adoWorkItemsDlpStatus[recordItem.fieldMapItem.dbField].status = DlpStatus.ISSUES_FOUND
+        adoWorkItemsDlpStatus[recordItem.fieldMapItem.dbField].issues.push({
+          text: piiMatch,
+          score: piiDetail.score
+        })
+        break
+      }
+    }
+  }
+  for (const statusField of Object.values(StatusFields)) {
+    if (adoWorkItemsDlpStatus[statusField].status !== DlpStatus.UNSCANNED) {
+      continue
+    }
+    adoWorkItemsDlpStatus[statusField].status = DlpStatus.NO_ISSUES
+    adoWorkItemsDlpStatus[statusField].issues = []
+  }
+  let dlpStatus = DlpStatus.NO_ISSUES
+  for (const statusField of Object.values(StatusFields)) {
+    if (adoWorkItemsDlpStatus[statusField].status === DlpStatus.ISSUES_FOUND) {
+      dlpStatus = DlpStatus.ISSUES_FOUND
+      break
+    }
+  }
+  adoWorkItemsDlpStatus.dlpStatus = dlpStatus
+  await adoWorkItemsDlpStatus.save()
+  return { message: 'OK' }
+}

DLPTrigger/lambda.ts

@@ -0,0 +1,64 @@
+import handleGetRequest from './handlers/get'
+import * as aws from 'aws-lambda'
+import handlePostRequest from './handlers/post'
+
+/**
+ * HTTP headers to be always returned to the client.
+ */
+const commonHeaders = {
+  'Access-Control-Allow-Credentials': 'true',
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Methods': 'GET, POST, OPTIONS, HANDSHAKE',
+  'Access-Control-Allow-Headers': '*',
+  'Access-Control-Max-Age': '86400',
+  // eslint-disable-next-line quote-props
+  'Vary': 'Accept-Encoding, Origin',
+  'Content-Type': 'application/json'
+}
+
+/**
+ * Entry point for the AWS lambda event handler.
+ * @param event AWS lambda event
+ * @param context AWS lambda context
+ */
+export async function handle (event: aws.APIGatewayEvent, context: aws.Context) {
+  const headers = commonHeaders
+  let statusCode = 200
+  let body: any = null
+
+  try {
+    switch (event.httpMethod) {
+      case 'GET': {
+        body = await handleGetRequest(event, context)
+        break
+      }
+      case 'POST': {
+        body = await handlePostRequest(event, context)
+        break
+      }
+      case 'OPTIONS':
+        // Nothing to do for OPTIONS method.
+        break
+      default:
+        statusCode = 405
+        body = { message: 'Unsupported Method.' }
+        break
+    }
+  } catch (err) {
+    if (err.isBoom) {
+      statusCode = err.output.statusCode
+      body = {
+        success: false,
+        message: err.message,
+        ...(err.data ? { data: err.data } : {})
+      }
+    } else {
+      console.error(`Error occurred: ${err.message as string}`)
+      console.error(err.stack)
+      statusCode = 500
+      body = { message: 'Internal Server Error' }
+    }
+  }
+
+  return { statusCode, headers, body: body ? JSON.stringify(body) : body }
+}