Add script to check AWS regions missing lambda layers (DataDog#73)

* Script to copy layers to new region

* Update list layers to output regions missing layers

* Run list layers before publishing to prod

* Fix condition logic

* Fix typo

* Return 0 if no version found for region

* Make script executable

Author: hghotra

scripts/add_new_region.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+
+# Copy layers from us-east-1 to new region
+# args: [new-region]
+
+set -e 
+
+OLD_REGION='us-east-1'
+
+PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
+LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
+NEW_REGION=$1
+
+publish_layer() {
+    region=$1
+    layer_name=$2
+    aws_version_key=$3
+    layer_path=$4
+
+    version_nbr=$(aws lambda publish-layer-version --layer-name $layer_name \
+        --description "Datadog Lambda Layer for Python" \
+        --zip-file "fileb://$layer_path" \
+        --region $region \
+        --compatible-runtimes $aws_version_key \
+                        | jq -r '.Version')
+
+    aws lambda add-layer-version-permission --layer-name $layer_name \
+        --version-number $version_nbr \
+        --statement-id "release-$version_nbr" \
+        --action lambda:GetLayerVersion --principal "*" \
+        --region $region
+
+    echo "Published layer for region $region, version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
+}
+
+get_max_version() {
+    layer_name=$1
+    region=$2
+    last_layer_version=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .Version")
+    if [ "$last_layer_version" == "null" ]; then
+        echo 0
+    else
+        echo $last_layer_version
+    fi
+}
+
+if [ -z "$1" ]; then
+    echo "Region parameter not specified, exiting"
+    exit 1
+fi
+
+j=0
+for layer_name in "${LAYER_NAMES[@]}"; do
+    # get latest version
+    last_layer_version=$(get_max_version $layer_name $OLD_REGION)
+    starting_version=$(get_max_version $layer_name $NEW_REGION)
+    starting_version=$(expr $starting_version + 1)
+
+    # exit if region is already all caught up
+    if [ $starting_version -gt $last_layer_version ]; then
+        echo "INFO: $NEW_REGION is already up to date for $layer_name"
+        continue
+    fi
+
+    # run for each version of layer
+    for i in $(seq 1 $last_layer_version); do 
+        layer_path=$layer_name"_"$i.zip
+        aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$j]}"
+
+        # download layer versions 
+        URL=$(AWS_REGION=$OLD_REGION aws lambda get-layer-version --layer-name $layer_name --version-number $i --query Content.Location --output text)
+        curl $URL -o $layer_path
+
+        # publish layer to new region
+        publish_layer $NEW_REGION $layer_name $aws_version_key $layer_path
+        rm $layer_path
+    done
+
+    j=$(expr $j + 1)
+done

scripts/list_layers.sh

@@ -8,20 +8,24 @@
 # Lists most recent layers ARNs across regions to STDOUT
 # Optionals args: [layer-name] [region]
 
+set -e
+
 LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
-AVAILABLE_REGIONS=(us-east-2 us-east-1 us-west-1 us-west-2 ap-east-1 ap-south-1 ap-northeast-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 ca-central-1 eu-north-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 sa-east-1)
+AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+LAYERS_MISSING_REGIONS=()
 
 # Check region arg
 if [ -z "$2" ]; then
     >&2 echo "Region parameter not specified, running for all available regions."
-    REGIONS=("${AVAILABLE_REGIONS[@]}")
+    REGIONS=$AVAILABLE_REGIONS
 else
+
     >&2 echo "Region parameter specified: $2"
-    if [[ ! " ${AVAILABLE_REGIONS[@]} " =~ " ${2} " ]]; then
-        >&2 echo "Could not find $2 in available regions: ${AVAILABLE_REGIONS[@]}"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$2"* ]]; then
+        >&2 echo "Could not find $2 in available regions:" $AVAILABLE_REGIONS
         >&2 echo ""
         >&2 echo "EXITING SCRIPT."
-        return 1
+        exit 1
     fi
     REGIONS=($2)
 fi
@@ -41,15 +45,24 @@ else
     LAYERS=($1)
 fi
 
-for region in "${REGIONS[@]}"
+for region in $REGIONS
 do
     for layer_name in "${LAYERS[@]}"
     do
         last_layer_arn=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .LayerVersionArn")
-        if [ -z $last_layer_arn ]; then
-             >&2 echo "No layer found for $region, $layer_name"
+        if [ "$last_layer_arn" == "null" ]; then
+            >&2 echo "No layer found for $region, $layer_name"
+            if [[ ! " ${LAYERS_MISSING_REGIONS[@]} " =~ " ${region} " ]]; then
+                LAYERS_MISSING_REGIONS+=( $region )
+            fi
         else
             echo $last_layer_arn
         fi
     done
 done
+
+if [ ${#LAYERS_MISSING_REGIONS[@]} -gt 0 ]; then
+    echo "WARNING: Following regions missing layers: ${LAYERS_MISSING_REGIONS[@]}"
+    echo "Please run ./add_new_region.sh <new_region> to add layers to the missing regions" 
+    exit 1
+fi

scripts/publish_staging.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
 
+./scripts/list_layers.sh
 ./scripts/build_layers.sh
 ./scripts/publish_layers.sh us-east-1