diff --git a/ci/datasources/environments.yaml b/ci/datasources/environments.yaml index 90056ab0..1ae2b4d7 100644 --- a/ci/datasources/environments.yaml +++ b/ci/datasources/environments.yaml @@ -1,9 +1,9 @@ environments: - - name: sandbox + sandbox: external_id: sandbox-publish-externalid role_to_assume: sandbox-layer-deployer account: 425362996713 - - name: prod + prod: external_id: prod-publish-externalid role_to_assume: dd-serverless-layer-deployer-role account: 464622532012 diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl index eae6b0d1..769f87f5 100644 --- a/ci/input_files/build.yaml.tpl +++ b/ci/input_files/build.yaml.tpl @@ -103,9 +103,6 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}): script: - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh -{{ range $environment := (ds "environments").environments }} - -{{ if or (eq $environment.name "prod") }} sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}): stage: sign tags: ["arch:amd64"] @@ -128,22 +125,25 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}): before_script: - apt-get update - apt-get install -y uuid-runtime + {{ with $environment := (ds "environments").environments.prod }} - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh + {{ end }} script: - - LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh {{ $environment.name }} -{{ end }} + - LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh prod + +{{ range $environment_name, $environment := (ds "environments").environments }} -publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}): +publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}): stage: publish tags: ["arch:amd64"] image: registry.ddbuild.io/images/docker:20.10-py3 rules: - - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' + - if: '"{{ $environment_name }}" == "sandbox"' when: manual allow_failure: true - if: '$CI_COMMIT_TAG =~ /^v.*/' needs: -{{ if or (eq $environment.name "prod") }} +{{ if or (eq $environment_name "prod") }} - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}}) {{ else }} - build-layer ({{ $runtime.name }}-{{ $runtime.arch }}) @@ -153,7 +153,7 @@ publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}): - integration-test ({{ $runtime.name }}-{{ $runtime.arch }}) {{ end }} dependencies: -{{ if or (eq $environment.name "prod") }} +{{ if or (eq $environment_name "prod") }} - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}}) {{ else }} - build-layer ({{ $runtime.name }}-{{ $runtime.arch }}) @@ -166,7 +166,7 @@ publish-layer-{{ $environment.name }} ({{ $runtime.name }}-{{ $runtime.arch }}): before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh script: - - STAGE={{ $environment.name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh + - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh {{- end }} @@ -186,3 +186,49 @@ publish-pypi-package: {{- end }} script: - ./ci/publish_pypi.sh + +layer bundle: + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + needs: + {{ range (ds "runtimes").runtimes }} + - build-layer ({{ .name }}-{{ .arch }}) + {{ end }} + dependencies: + {{ range (ds "runtimes").runtimes }} + - build-layer ({{ .name }}-{{ .arch }}) + {{ end }} + artifacts: + expire_in: 1 hr + paths: + - datadog_lambda_py-bundle-${CI_JOB_ID}/ + name: datadog_lambda_py-bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_py-bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID} + +signed layer bundle: + stage: sign + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + {{ range (ds "runtimes").runtimes }} + - sign-layer ({{ .name }}-{{ .arch }}) + {{ end }} + dependencies: + {{ range (ds "runtimes").runtimes }} + - sign-layer ({{ .name }}-{{ .arch }}) + {{ end }} + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_py-signed-bundle-${CI_JOB_ID}/ + name: datadog_lambda_py-signed-bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID} diff --git a/ci/publish_layers.sh b/ci/publish_layers.sh index 85317ddd..58257bf1 100755 --- a/ci/publish_layers.sh +++ b/ci/publish_layers.sh @@ -24,7 +24,20 @@ AWS_CLI_PYTHON_VERSIONS=( "python3.13" "python3.13" ) -PYTHON_VERSIONS=("3.8-amd64" "3.8-arm64" "3.9-amd64" "3.9-arm64" "3.10-amd64" "3.10-arm64" "3.11-amd64" "3.11-arm64" "3.12-amd64" "3.12-arm64" "3.13-amd64" "3.13-arm64") +PYTHON_VERSIONS=( + "3.8-amd64" + "3.8-arm64" + "3.9-amd64" + "3.9-arm64" + "3.10-amd64" + "3.10-arm64" + "3.11-amd64" + "3.11-arm64" + "3.12-amd64" + "3.12-arm64" + "3.13-amd64" + "3.13-arm64" +) LAYER_PATHS=( ".layers/datadog_lambda_py-amd64-3.8.zip" ".layers/datadog_lambda_py-arm64-3.8.zip" @@ -53,11 +66,16 @@ LAYERS=( "Datadog-Python313" "Datadog-Python313-ARM" ) -STAGES=('prod', 'sandbox', 'staging') +STAGES=('prod', 'sandbox', 'staging', 'gov-staging', 'gov-prod') printf "Starting script...\n\n" -printf "Installing dependencies\n" -pip install awscli + +if [ -z "$SKIP_PIP_INSTALL" ]; then + echo "Installing dependencies" + pip install awscli +else + echo "Skipping pip install" +fi publish_layer() { region=$1 @@ -89,7 +107,7 @@ fi printf "Python version specified: $PYTHON_VERSION\n" if [[ ! ${PYTHON_VERSIONS[@]} =~ $PYTHON_VERSION ]]; then - printf "[Error] Unsupported PYTHON_VERSION found.\n" + printf "[Error] Unsupported PYTHON_VERSION found: $PYTHON_VERSION.\n" exit 1 fi @@ -133,8 +151,14 @@ if [[ ! ${STAGES[@]} =~ $STAGE ]]; then fi layer="${LAYERS[$index]}" +if [ -z "$LAYER_NAME_SUFFIX" ]; then + echo "No layer name suffix" +else + layer="${layer}-${LAYER_NAME_SUFFIX}" +fi +echo "layer name: $layer" -if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then +if [[ "$STAGE" =~ ^(staging|sandbox|gov-staging)$ ]]; then # Deploy latest version latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $layer --query 'LayerVersions[0].Version || `0`') VERSION=$(($latest_version + 1)) diff --git a/scripts/publish_govcloud.sh b/scripts/publish_govcloud.sh new file mode 100755 index 00000000..5fd107b0 --- /dev/null +++ b/scripts/publish_govcloud.sh @@ -0,0 +1,105 @@ +#! /usr/bin/env bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2025 Datadog, Inc. +# +# USAGE: download the layer bundle from the build pipeline in gitlab. Use the +# Download button on the `layer bundle` job. This will be a zip file containing +# all of the required layers. Run this script as follows: +# +# ENVIRONMENT=[us1-staging-fed or us1-fed] [LAYER_NAME_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud.sh +# +# protip: you can drag the zip file from finder into your terminal to insert +# its path. + +set -e + +LAYER_PACKAGE=$1 + +if [ -z "$LAYER_PACKAGE" ]; then + printf "[ERROR]: layer package not provided\n" + exit 1 +fi + +PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip) + +if [ -z "$ENVIRONMENT" ]; then + printf "[ERROR]: ENVIRONMENT not specified\n" + exit 1 +fi + +if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user + + export STAGE=gov-staging + + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_py-(signed-)?bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +elif [ $ENVIRONMENT = "us1-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering + + export STAGE=gov-prod + + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_py-signed-bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +else + printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n" + exit 1 +fi + +TEMP_DIR=$(mktemp -d) +unzip $LAYER_PACKAGE -d $TEMP_DIR +cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/ + + +AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --" + +echo "Checking that you have access to the GovCloud AWS account" +$AWS_VAULT_PREFIX aws sts get-caller-identity + + +AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +# Determine the target regions +if [ -z "$REGIONS" ]; then + echo "Region not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + echo "Region specified: $REGIONS" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then + echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS" + echo "" + echo "EXITING SCRIPT." + exit 1 + fi +fi + +for region in $REGIONS +do + echo "Starting publishing layers for region $region..." + + export REGION=$region + + for python_version in "3.8" "3.9" "3.10" "3.11" "3.12" "3.13"; do + for arch in "amd64" "arm64"; do + export PYTHON_VERSION=$python_version + export ARCH=$arch + + export SKIP_PIP_INSTALL=true + + echo "Publishing layer for $PYTHON_VERSION and $ARCH" + + $AWS_VAULT_PREFIX ./ci/publish_layers.sh + done + done +done + +echo "Done !" diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index f3d13653..d2918c54 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -6,6 +6,11 @@ set -e read -p "Are we only doing the simplified GovCloud release? ONLY IF THE NORMAL RELEASE IS DONE AND YOU HAVE DOWNLOADED THE LAYERS (y/n)? " GOVCLOUD_ONLY +if [ $GOVCLOUD_ONLY != "n" ]; then + echo "GovCloud publishing is now supported only in publich_govcloud.sh" + exit 1 +fi + # Ensure on main, and pull the latest BRANCH=$(git rev-parse --abbrev-ref HEAD) if [ $BRANCH != "main" ]; then @@ -34,107 +39,82 @@ else fi # Ensure pypi registry access -if [ $GOVCLOUD_ONLY == "y" ]; then - echo "Skipping PyPI check since this is a GovCloud-only release" - -else - read -p "Do you have access to PyPI (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 - fi +read -p "Do you have access to PyPI (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 fi CURRENT_VERSION=$(poetry version --short) LAYER_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2) -if [ $GOVCLOUD_ONLY == "y" ]; then - echo "Skipping Libary Updates, code changes, layer builds and signing for GovCloud-only release" +read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi +echo "Answer 'n' if already done in a PR" +read -p "Update pyproject.toml version? (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Skipping updating package.json version" else - read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 - fi - - echo "Answer 'n' if already done in a PR" - read -p "Update pyproject.toml version? (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Skipping updating package.json version" - else - echo - echo "Replacing version in pyproject.toml and datadog_lambda/version.py" - echo - - poetry version ${NEW_VERSION} - echo "__version__ = \"${NEW_VERSION}\"" > datadog_lambda/version.py - fi - echo - echo "Building layers..." - ./scripts/build_layers.sh - + echo "Replacing version in pyproject.toml and datadog_lambda/version.py" echo - echo "Signing layers for commercial AWS regions" - aws-vault exec sso-prod-engineering -- ./scripts/sign_layers.sh prod - - echo "Answer 'n' if GitLab already did this" - read -p "Deploy layers to commercial AWS (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Skipping deployment to commercial AWS" - else - echo "Ensuring you have access to the production AWS account" - aws-vault exec sso-prod-engineering -- aws sts get-caller-identity - - echo - echo "Publishing layers to commercial AWS regions" - VERSION=$LAYER_VERSION aws-vault exec sso-prod-engineering --no-session -- ./scripts/publish_layers.sh - fi + + poetry version ${NEW_VERSION} + echo "__version__ = \"${NEW_VERSION}\"" > datadog_lambda/version.py fi -read -p "Deploy layers to GovCloud AWS (y/n)?" CONT +echo +echo "Building layers..." +./scripts/build_layers.sh + +echo +echo "Signing layers for commercial AWS regions" +aws-vault exec sso-prod-engineering -- ./scripts/sign_layers.sh prod + +echo "Answer 'n' if GitLab already did this" +read -p "Deploy layers to commercial AWS (y/n)?" CONT if [ "$CONT" != "y" ]; then - echo "Skipping deployment to GovCloud AWS" + echo "Skipping deployment to commercial AWS" else - echo "Ensuring you have access to the AWS GovCloud account" - aws-vault exec sso-govcloud-us1-fed-engineering -- aws sts get-caller-identity + echo "Ensuring you have access to the production AWS account" + aws-vault exec sso-prod-engineering -- aws sts get-caller-identity - echo "Publishing layers to GovCloud AWS regions" - VERSION=$LAYER_VERSION aws-vault exec sso-govcloud-us1-fed-engineering -- ./scripts/publish_layers.sh + echo + echo "Publishing layers to commercial AWS regions" + VERSION=$LAYER_VERSION aws-vault exec sso-prod-engineering --no-session -- ./scripts/publish_layers.sh fi -if [ $GOVCLOUD_ONLY == "y" ]; then - echo "Skipping PyPI check and Github Release since this is a GovCloud-only release" - +echo "Answer 'n' if GitLab already did this" +read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Skipping publishing to PyPI" else - echo "Answer 'n' if GitLab already did this" - read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Skipping publishing to PyPI" - else - echo - echo "Publishing to https://pypi.org/project/datadog-lambda/" - ./scripts/pypi.sh - fi - - - echo "Answer 'n' if you already released in GitHub" - read -p "Do you want to bump the version in GitHub? (y/n)" CONT - if [ "$CONT" != "y" ]; then - echo "Skipping publishing updates to GitHub" - else - echo - echo 'Publishing updates to github' - git commit pyproject.toml datadog_lambda/version.py -m "Bump version to ${NEW_VERSION}" - git push origin main - git tag "v$LAYER_VERSION" - git push origin "refs/tags/v$LAYER_VERSION" - fi + echo + echo "Publishing to https://pypi.org/project/datadog-lambda/" + ./scripts/pypi.sh +fi + +echo "Answer 'n' if you already released in GitHub" +read -p "Do you want to bump the version in GitHub? (y/n)" CONT +if [ "$CONT" != "y" ]; then + echo "Skipping publishing updates to GitHub" +else echo - echo "Now create a new release with the tag v${LAYER_VERSION} created unless you have done this already" - echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION" + echo 'Publishing updates to github' + git commit pyproject.toml datadog_lambda/version.py -m "Bump version to ${NEW_VERSION}" + git push origin main + git tag "v$LAYER_VERSION" + git push origin "refs/tags/v$LAYER_VERSION" fi + +echo +echo "Now create a new release with the tag v${LAYER_VERSION} created unless you have done this already" +echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION" + # Open a PR to the documentation repo to automatically bump layer version VERSION=$LAYER_VERSION LAYER=datadog-lambda-python ./scripts/create_documentation_pr.sh