Darcy.rayner/add arm publishing (DataDog#182)

* Add arm publishing code

* Fix issues with ARM 3.9 config in build scripts

* Update integration tests

* Add cross-compiling deps to ubuntu build job

* Fix install

* Fix check-size

Author: DarcyRaynerDD

.github/workflows/build.yml

@@ -83,6 +83,8 @@ jobs:
 
       - name: Install Serverless Framework
         run: sudo yarn global add serverless --prefix /usr/local
+      - name: Install Crossbuild Deps
+        run: sudo apt install -y qemu-user-static binfmt-support
 
       - name: Install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'

.github/workflows/check-size.yml

@@ -14,6 +14,9 @@ jobs:
         with:
           python-version: 3.7
 
+      - name: Install Crossbuild Deps
+        run: sudo apt install -y qemu-user-static binfmt-support
+
       - name: Install dependencies
         run: |
           pip install virtualenv

scripts/build_layers.sh

@@ -40,17 +40,21 @@ function docker_build_zip {
     # Args: [python version] [zip destination]
 
     destination=$(make_path_absolute $2)
+    arch=$3
 
     # Install datadogpy in a docker container to avoid the mess from switching
     # between different python runtimes.
     temp_dir=$(mktemp -d)
-    docker build -t datadog-lambda-python:$1 . --no-cache \
+    docker buildx build -t datadog-lambda-python-${arch}:$1 . --no-cache \
         --build-arg image=python:$1 \
-        --build-arg runtime=python$1
+        --build-arg runtime=python$1 \
+        --platform linux/${arch} \
+        --load
 
     # Run the image by runtime tag, tar its generatd `python` directory to sdout,
     # then extract it to a temp directory.
-    docker run datadog-lambda-python:$1 tar cf - python | tar -xf - -C $temp_dir
+    docker run datadog-lambda-python-${arch}:$1 tar cf - python | tar -xf - -C $temp_dir
+
 
     # Zip to destination, and keep directory structure as based in $temp_dir
     (cd $temp_dir && zip -q -r $destination ./)
@@ -64,8 +68,12 @@ mkdir $LAYER_DIR
 
 for python_version in "${PYTHON_VERSIONS[@]}"
 do
-    echo "Building layer for Python ${python_version}"
-    docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}${python_version}.zip
+    if [ "$python_version" == "3.8" ] || [ "$python_version" == "3.9" ]; then
+        echo "Building layer for Python ${python_version} arch=arm64"
+        docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-arm64-${python_version}.zip arm64
+    fi
+    echo "Building layer for Python ${python_version} arch=amd64"
+    docker_build_zip ${python_version} $LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${python_version}.zip amd64
 done
 
 echo "Done creating layers:"

scripts/check_layer_size.sh

@@ -18,7 +18,7 @@ VERSIONS=("2.7" "3.6" "3.7" "3.8" "3.9")
 
 for version in "${VERSIONS[@]}"
 do
-    FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}${version}.zip
+    FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-amd64-${version}.zip
     FILE_SIZE=$(stat --printf="%s" $FILE)
     FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))"
     echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb"

scripts/list_layers.sh

@@ -10,7 +10,7 @@
 
 set -e
 
-LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
+LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 LAYERS_MISSING_REGIONS=()
 

scripts/publish_layers.sh

@@ -13,9 +13,10 @@ set -e
 # Makes sure any subprocesses will be terminated with this process
 trap "pkill -P $$; exit 1;" INT
 
-PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.9")
-LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip" ".layers/datadog_lambda_py3.9.zip")
-AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python39")
+PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8" "python3.8" "python3.9" "python3.9")
+LAYER_PATHS=(".layers/datadog_lambda_py-amd64-2.7.zip" ".layers/datadog_lambda_py-amd64-3.6.zip" ".layers/datadog_lambda_py-amd64-3.7.zip" ".layers/datadog_lambda_py-amd64-3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" ".layers/datadog_lambda_py-amd64-3.9.zip" ".layers/datadog_lambda_py-arm64-3.9.zip")
+AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38" "Datadog-Python38-ARM" "Datadog-Python39" "Datadog-Python39-ARM")
+ARCHS=("amd64" "amd64" "amd64""amd64" "amd64" "arm64")
 AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
 
 # Check that the layer files exist

scripts/sign_layers.sh

@@ -9,11 +9,13 @@ set -e
 
 LAYER_DIR=".layers"
 LAYER_FILES=(
-    "datadog_lambda_py2.7.zip"
-    "datadog_lambda_py3.6.zip"
-    "datadog_lambda_py3.7.zip"
-    "datadog_lambda_py3.8.zip"
-    "datadog_lambda_py3.9.zip"
+    "datadog_lambda_py-amd64-2.7.zip"
+    "datadog_lambda_py-amd64-3.6.zip"
+    "datadog_lambda_py-amd64-3.7.zip"
+    "datadog_lambda_py-amd64-3.8.zip"
+    "datadog_lambda_py-arm64-3.8.zip"
+    "datadog_lambda_py-amd64-3.9.zip"
+    "datadog_lambda_py-arm64-3.9.zip"
 )
 SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile"
 

tests/integration/serverless.yml

@@ -27,7 +27,7 @@ provider:
 layers:
   python:
     package:
-      artifact: ../../.layers/datadog_lambda_py${env:PYTHON_VERSION}.zip
+      artifact: ../../.layers/datadog_lambda_py-amd64-${env:PYTHON_VERSION}.zip
 
 functions:
   # async-metrics (flushed to logs)
@@ -46,4 +46,4 @@ functions:
     handler: handle.handle
     runtime: ${env:SERVERLESS_RUNTIME}
     layers:
-      - { Ref: PythonLambdaLayer }
+      - { Ref: PythonLambdaLayer }