From 42aee21cf8ce4bee56cd083cee2446a1c33ba2a9 Mon Sep 17 00:00:00 2001 From: Renato Valenzuela <37676028+valerena@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:56:46 -0700 Subject: [PATCH 01/12] chore(deps): Update to Go 1.21. Update deps (#116) --- Makefile | 2 +- go.mod | 15 +++++++-------- go.sum | 30 ++++++++++++------------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 80ccb89..1916dae 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ compile-lambda-linux-all: make ARCH=old compile-lambda-linux compile-with-docker: - docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.20 make ARCH=${ARCH} compile-lambda-linux + docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.21 make ARCH=${ARCH} compile-lambda-linux compile-lambda-linux: CGO_ENABLED=0 GOOS=linux GOARCH=${GO_ARCH_${ARCH}} go build -buildvcs=false -ldflags "${RELEASE_BUILD_LINKER_FLAGS}" -o ${DESTINATION_${ARCH}} ./cmd/aws-lambda-rie diff --git a/go.mod b/go.mod index 990a7dd..d48dc60 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,21 @@ module go.amzn.com -go 1.20 +go 1.21 require ( - github.com/aws/aws-lambda-go v1.41.0 - github.com/go-chi/chi v4.1.2+incompatible - github.com/google/uuid v1.3.0 + github.com/aws/aws-lambda-go v1.46.0 + github.com/go-chi/chi v1.5.5 + github.com/google/uuid v1.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 - golang.org/x/sync v0.2.0 + github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.6.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - golang.org/x/net v0.18.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect golang.org/x/sys v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0ea11d6..8974775 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,12 @@ -github.com/aws/aws-lambda-go v1.41.0 h1:l/5fyVb6Ud9uYd411xdHZzSf2n86TakxzpvIoz7l+3Y= -github.com/aws/aws-lambda-go v1.41.0/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/aws/aws-lambda-go v1.46.0 h1:UWVnvh2h2gecOlFhHQfIPQcD8pL/f7pVCutmFl+oXU8= +github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= -github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= +github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -14,23 +14,17 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4c2c20dfc5b09f6e93bb49a31bb62a6ad628c262 Mon Sep 17 00:00:00 2001 From: Renato Valenzuela <37676028+valerena@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:29:07 -0700 Subject: [PATCH 02/12] chore(deps): Update to Go 1.22 (#117) * Update to Go 1.22 * Update Makefile to run unit tests in container --- Makefile | 13 ++++++++----- go.mod | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 1916dae..6d36ae2 100644 --- a/Makefile +++ b/Makefile @@ -10,10 +10,10 @@ DESTINATION_old:= bin/${BINARY_NAME} DESTINATION_x86_64 := bin/${BINARY_NAME}-x86_64 DESTINATION_arm64 := bin/${BINARY_NAME}-arm64 +run_in_docker = docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.22 $(1) + compile-with-docker-all: - make ARCH=x86_64 compile-with-docker - make ARCH=arm64 compile-with-docker - make ARCH=old compile-with-docker + $(call run_in_docker, make compile-lambda-linux-all) compile-lambda-linux-all: make ARCH=x86_64 compile-lambda-linux @@ -21,11 +21,14 @@ compile-lambda-linux-all: make ARCH=old compile-lambda-linux compile-with-docker: - docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.21 make ARCH=${ARCH} compile-lambda-linux + $(call run_in_docker, make ARCH=${ARCH} compile-lambda-linux) compile-lambda-linux: CGO_ENABLED=0 GOOS=linux GOARCH=${GO_ARCH_${ARCH}} go build -buildvcs=false -ldflags "${RELEASE_BUILD_LINKER_FLAGS}" -o ${DESTINATION_${ARCH}} ./cmd/aws-lambda-rie +tests-with-docker: + $(call run_in_docker, make tests) + tests: go test ./... @@ -33,7 +36,7 @@ integ-tests-and-compile: tests make compile-lambda-linux-all make integ-tests -integ-tests-with-docker: tests +integ-tests-with-docker: tests-with-docker make compile-with-docker-all make integ-tests diff --git a/go.mod b/go.mod index d48dc60..4ee45d7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.amzn.com -go 1.21 +go 1.22 require ( github.com/aws/aws-lambda-go v1.46.0 From fe11d78c4ba308bcee3963b237a91ea8c746d439 Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Fri, 19 Apr 2024 13:58:03 -0700 Subject: [PATCH 03/12] test: Refactor test cases (#119) Refactor testcases to - Use python3.12 - Respect docker architecture - Run different architecture in parallel GitHub actions --- .github/workflows/integ-tests.yml | 38 ++- Makefile | 30 +- .../local_lambda/test_end_to_end.py | 289 +++++++++--------- test/integration/testdata/Dockerfile-allinone | 3 +- 4 files changed, 211 insertions(+), 149 deletions(-) diff --git a/.github/workflows/integ-tests.yml b/.github/workflows/integ-tests.yml index cb2f9dc..7fddc95 100644 --- a/.github/workflows/integ-tests.yml +++ b/.github/workflows/integ-tests.yml @@ -6,16 +6,44 @@ on: - develop jobs: - integ-tests: + go-tests: runs-on: ubuntu-latest environment: - name: prod + name: integ-tests + steps: + - uses: actions/checkout@v4 + - name: run go tests + run: make tests-with-docker + integ-tests-x86: + runs-on: ubuntu-latest + environment: + name: integ-tests + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: run integration tests + run: make integ-tests-with-docker-x86-64 + integ-tests-arm64: + runs-on: ubuntu-latest + environment: + name: integ-tests + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: run integration tests + run: make integ-tests-with-docker-arm64 + integ-tests-old: + runs-on: ubuntu-latest + environment: + name: integ-tests steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: allows us to build arm64 images - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: run integration tests - run: make integ-tests-with-docker \ No newline at end of file + run: make integ-tests-with-docker-old \ No newline at end of file diff --git a/Makefile b/Makefile index 6d36ae2..f7a714e 100644 --- a/Makefile +++ b/Makefile @@ -39,9 +39,35 @@ integ-tests-and-compile: tests integ-tests-with-docker: tests-with-docker make compile-with-docker-all make integ-tests - -integ-tests: + +prep-python: python3 -m venv .venv .venv/bin/pip install --upgrade pip .venv/bin/pip install requests parameterized + +exec-python-e2e-test: .venv/bin/python3 test/integration/local_lambda/test_end_to_end.py + +integ-tests: + make prep-python + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + make TEST_ARCH=x86_64 TEST_PORT=8002 exec-python-e2e-test + make TEST_ARCH=arm64 TEST_PORT=9002 exec-python-e2e-test + make TEST_ARCH="" TEST_PORT=9052 exec-python-e2e-test + +integ-tests-with-docker-x86-64: + make ARCH=x86_64 compile-with-docker + make prep-python + make TEST_ARCH=x86_64 TEST_PORT=8002 exec-python-e2e-test + +integ-tests-with-docker-arm64: + make ARCH=arm64 compile-with-docker + make prep-python + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + make TEST_ARCH=arm64 TEST_PORT=9002 exec-python-e2e-test + +integ-tests-with-docker-old: + make ARCH=old compile-with-docker + make prep-python + make TEST_ARCH="" TEST_PORT=9052 exec-python-e2e-test + \ No newline at end of file diff --git a/test/integration/local_lambda/test_end_to_end.py b/test/integration/local_lambda/test_end_to_end.py index fd7f735..7c5486f 100644 --- a/test/integration/local_lambda/test_end_to_end.py +++ b/test/integration/local_lambda/test_end_to_end.py @@ -5,73 +5,57 @@ from unittest import TestCase, main from pathlib import Path import time - +import os import requests +from contextlib import contextmanager from parameterized import parameterized -SLEEP_TIME = 2 +SLEEP_TIME = 1.5 DEFAULT_1P_ENTRYPOINT = "/lambda-entrypoint.sh" ARCHS = ["x86_64", "arm64", ""] + class TestEndToEnd(TestCase): + ARCH = os.environ.get('TEST_ARCH', "") + PORT = os.environ.get('TEST_PORT', 8002) @classmethod def setUpClass(cls): testdata_path = Path(__file__).resolve().parents[1].joinpath("testdata") dockerfile_path = testdata_path.joinpath("Dockerfile-allinone") - cls.image_name = "aws-lambda-local:testing" cls.path_to_binary = Path().resolve().joinpath("bin") # build image - for arch in ARCHS: - image_name = cls.image_name if arch == "" else f"{cls.image_name}-{arch}" - architecture = arch if arch == "arm64" else "amd64" - build_cmd = [ - "docker", - "build", - "--platform", - f"linux/{architecture}", - "-t", - image_name, - "-f", - str(dockerfile_path), - str(testdata_path), - ] - Popen(build_cmd).communicate() + image_name_base = "aws-lambda-local:testing" + cls.image_name = image_name_base if cls.ARCH == "" else f"{image_name_base}-{cls.ARCH}" + architecture = cls.ARCH if cls.ARCH == "arm64" else "amd64" + docker_arch = cls.ARCH if cls.ARCH == "arm64" else "x86_64" + + build_cmd = [ + "docker", + "build", + "--platform", + f"linux/{architecture}", + "-t", + cls.image_name, + "-f", + str(dockerfile_path), + str(testdata_path), + "--build-arg", + f"IMAGE_ARCH={docker_arch}", + ] + Popen(build_cmd).communicate() @classmethod def tearDownClass(cls): - images_to_delete = [ - "envvarcheck", - "twoinvokes", - "arnexists", - "customname", - "timeout", - "exception", - "remaining_time_in_three_seconds", - "remaining_time_in_ten_seconds", - "remaining_time_in_default_deadline", - "pre-runtime-api", - "assert-overwritten", - "port_override" - ] - - for image in images_to_delete: - for arch in ARCHS: - arch_tag = "" if arch == "" else f"-{arch}" - cmd = f"docker rm -f {image}{arch_tag}" - Popen(cmd.split(" ")).communicate() - - for arch in ARCHS: - arch_tag = "" if arch == "" else f"-{arch}" - Popen(f"docker rmi {cls.image_name}{arch_tag}".split(" ")).communicate() + Popen(f"docker rmi {cls.image_name}".split(" ")).communicate() - def tagged_name(self, name, architecture): - tag = self.get_tag(architecture) - return (name + tag, "aws-lambda-rie" + tag, self.image_name + tag) + def tagged_name(self, name): + tag = self.get_tag() + return (name + tag, "aws-lambda-rie" + tag, self.image_name) - def get_tag(self, architecture): - return "" if architecture == "" else str(f"-{architecture}") + def get_tag(self): + return "" if self.ARCH == "" else str(f"-{self.ARCH}") def run_command(self, cmd): Popen(cmd.split(" ")).communicate() @@ -79,153 +63,176 @@ def run_command(self, cmd): def sleep_1s(self): time.sleep(SLEEP_TIME) - def invoke_function(self, port): + def invoke_function(self): return requests.post( - f"http://localhost:{port}/2015-03-31/functions/function/invocations", json={} + f"http://localhost:{self.PORT}/2015-03-31/functions/function/invocations", json={} ) - def create_container_and_invoke_function(self, cmd, port): - self.run_command(cmd) + @contextmanager + def create_container(self, param, image): + try: + platform = "x86_64" if self.ARCH == "" else self.ARCH + cmd_full = f"docker run --platform linux/{platform} {param}" + self.run_command(cmd_full) - # sleep 1s to give enough time for the endpoint to be up to curl - self.sleep_1s() + # sleep 1s to give enough time for the endpoint to be up to curl + self.sleep_1s() + yield + except Exception as e: + print(f"An error occurred while executing cmd: {cmd_full}. error: {e}") + raise e + finally: + self.run_command(f"docker stop {image}") + self.run_command(f"docker rm -f {image}") - return self.invoke_function(port) - @parameterized.expand([("x86_64", "8000"), ("arm64", "9000"), ("", "9050")]) - def test_env_var_with_equal_sign(self, arch, port): - image, rie, image_name = self.tagged_name("envvarcheck", arch) - - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_env_var_handler" + def test_env_var_with_equal_sign(self): + image, rie, image_name = self.tagged_name("envvarcheck") + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_env_var_handler" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"4=4"', r.content) + self.assertEqual(b'"4=4"', r.content) - @parameterized.expand([("x86_64", "8001"), ("arm64", "9001"), ("", "9051")]) - def test_two_invokes(self, arch, port): - image, rie, image_name = self.tagged_name("twoinvokes", arch) - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler" + def test_two_invokes(self): + image, rie, image_name = self.tagged_name("twoinvokes") - r = self.create_container_and_invoke_function(cmd, port) - - self.assertEqual(b'"My lambda ran succesfully"', r.content) + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler" - # Make sure we can invoke the function twice - r = self.invoke_function(port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) + + # Make sure we can invoke the function twice + r = self.invoke_function() + + self.assertEqual(b'"My lambda ran succesfully"', r.content) + - @parameterized.expand([("x86_64", "8002"), ("arm64", "9002"), ("", "9052")]) - def test_lambda_function_arn_exists(self, arch, port): - image, rie, image_name = self.tagged_name("arnexists", arch) + def test_lambda_function_arn_exists(self): + image, rie, image_name = self.tagged_name("arnexists") - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_lambda_arn_in_context" + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_lambda_arn_in_context" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) + - @parameterized.expand([("x86_64", "8003"), ("arm64", "9003"), ("", "9053")]) - def test_lambda_function_arn_exists_with_defining_custom_name(self, arch, port): - image, rie, image_name = self.tagged_name("customname", arch) + def test_lambda_function_arn_exists_with_defining_custom_name(self): + image, rie, image_name = self.tagged_name("customname") - cmd = f"docker run --name {image} --env AWS_LAMBDA_FUNCTION_NAME=MyCoolName -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_lambda_arn_in_context" + params = f"--name {image} --env AWS_LAMBDA_FUNCTION_NAME=MyCoolName -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_lambda_arn_in_context" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) - @parameterized.expand([("x86_64", "8004"), ("arm64", "9004"), ("", "9054")]) - def test_timeout_invoke(self, arch, port): - image, rie, image_name = self.tagged_name("timeout", arch) - cmd = f"docker run --name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=1 -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.sleep_handler" + def test_timeout_invoke(self): + image, rie, image_name = self.tagged_name("timeout") - r = self.create_container_and_invoke_function(cmd, port) + params = f"--name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=1 -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.sleep_handler" + + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b"Task timed out after 1.00 seconds", r.content) + self.assertEqual(b"Task timed out after 1.00 seconds", r.content) - @parameterized.expand([("x86_64", "8005"), ("arm64", "9005"), ("", "9055")]) - def test_exception_returned(self, arch, port): - image, rie, image_name = self.tagged_name("exception", arch) - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.exception_handler" + def test_exception_returned(self): + image, rie, image_name = self.tagged_name("exception") + + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.exception_handler" + + with self.create_container(params, image): + r = self.invoke_function() + + # Except the 3 fields assrted below, there's another field `request_id` included start from python3.12 runtime. + # We should ignore asserting the field `request_id` for it is in a UUID like format and changes everytime + result = r.json() + self.assertEqual(result["errorMessage"], "Raising an exception") + self.assertEqual(result["errorType"], "Exception") + self.assertEqual(result["stackTrace"], [" File \"/var/task/main.py\", line 13, in exception_handler\n raise Exception(\"Raising an exception\")\n"]) - r = self.create_container_and_invoke_function(cmd, port) - - self.assertEqual( - b'{"errorMessage": "Raising an exception", "errorType": "Exception", "stackTrace": [" File \\"/var/task/main.py\\", line 13, in exception_handler\\n raise Exception(\\"Raising an exception\\")\\n"]}', - r.content, - ) - @parameterized.expand([("x86_64", "8006"), ("arm64", "9006"), ("", "9056")]) - def test_context_get_remaining_time_in_three_seconds(self, arch, port): - image, rie, image_name = self.tagged_name("remaining_time_in_three_seconds", arch) + def test_context_get_remaining_time_in_three_seconds(self): + image, rie, image_name = self.tagged_name("remaining_time_in_three_seconds") - cmd = f"docker run --name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=3 -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" + params = f"--name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=3 -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - # Execution time is not decided, 1.0s ~ 3.0s is a good estimation - self.assertLess(int(r.content), 3000) - self.assertGreater(int(r.content), 1000) + # Execution time is not decided, 1.0s ~ 3.0s is a good estimation + self.assertLess(int(r.content), 3000) + self.assertGreater(int(r.content), 1000) - @parameterized.expand([("x86_64", "8007"), ("arm64", "9007"), ("", "9057")]) - def test_context_get_remaining_time_in_ten_seconds(self, arch, port): - image, rie, image_name = self.tagged_name("remaining_time_in_ten_seconds", arch) - cmd = f"docker run --name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=10 -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" + def test_context_get_remaining_time_in_ten_seconds(self): + image, rie, image_name = self.tagged_name("remaining_time_in_ten_seconds") - r = self.create_container_and_invoke_function(cmd, port) + params = f"--name {image} -d --env AWS_LAMBDA_FUNCTION_TIMEOUT=10 -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" + + with self.create_container(params, image): + r = self.invoke_function() - # Execution time is not decided, 8.0s ~ 10.0s is a good estimation - self.assertLess(int(r.content), 10000) - self.assertGreater(int(r.content), 8000) + # Execution time is not decided, 8.0s ~ 10.0s is a good estimation + self.assertLess(int(r.content), 10000) + self.assertGreater(int(r.content), 8000) + + + def test_context_get_remaining_time_in_default_deadline(self): + image, rie, image_name = self.tagged_name("remaining_time_in_default_deadline") - @parameterized.expand([("x86_64", "8008"), ("arm64", "9008"), ("", "9058")]) - def test_context_get_remaining_time_in_default_deadline(self, arch, port): - image, rie, image_name = self.tagged_name("remaining_time_in_default_deadline", arch) + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.check_remaining_time_handler" + with self.create_container(params, image): + r = self.invoke_function() - r = self.create_container_and_invoke_function(cmd, port) + # Executation time is not decided, 298.0s ~ 300.0s is a good estimation + self.assertLess(int(r.content), 300000) + self.assertGreater(int(r.content), 298000) - # Executation time is not decided, 298.0s ~ 300.0s is a good estimation - self.assertLess(int(r.content), 300000) - self.assertGreater(int(r.content), 298000) - @parameterized.expand([("x86_64", "8009"), ("arm64", "9009"), ("", "9059")]) - def test_invoke_with_pre_runtime_api_runtime(self, arch, port): - image, rie, image_name = self.tagged_name("pre-runtime-api", arch) + def test_invoke_with_pre_runtime_api_runtime(self): + image, rie, image_name = self.tagged_name("pre-runtime-api") - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler" + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) - @parameterized.expand([("x86_64", "8010"), ("arm64", "9010"), ("", "9060")]) - def test_function_name_is_overriden(self, arch, port): - image, rie, image_name = self.tagged_name("assert-overwritten", arch) - cmd = f"docker run --name {image} -d --env AWS_LAMBDA_FUNCTION_NAME=MyCoolName -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_env_var_is_overwritten" + def test_function_name_is_overriden(self): + image, rie, image_name = self.tagged_name("assert-overwritten") - r = self.create_container_and_invoke_function(cmd, port) + params = f"--name {image} -d --env AWS_LAMBDA_FUNCTION_NAME=MyCoolName -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.assert_env_var_is_overwritten" + + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) + - @parameterized.expand([("x86_64", "8011"), ("arm64", "9011"), ("", "9061")]) - def test_port_override(self, arch, port): - image, rie, image_name = self.tagged_name("port_override", arch) + def test_port_override(self): + image, rie, image_name = self.tagged_name("port_override") # Use port 8081 inside the container instead of 8080 - cmd = f"docker run --name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {port}:8081 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler --runtime-interface-emulator-address 0.0.0.0:8081" + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8081 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.success_handler --runtime-interface-emulator-address 0.0.0.0:8081" - r = self.create_container_and_invoke_function(cmd, port) + with self.create_container(params, image): + r = self.invoke_function() - self.assertEqual(b'"My lambda ran succesfully"', r.content) + self.assertEqual(b'"My lambda ran succesfully"', r.content) + if __name__ == "__main__": diff --git a/test/integration/testdata/Dockerfile-allinone b/test/integration/testdata/Dockerfile-allinone index b804e5c..1d28406 100644 --- a/test/integration/testdata/Dockerfile-allinone +++ b/test/integration/testdata/Dockerfile-allinone @@ -1,4 +1,5 @@ -FROM public.ecr.aws/lambda/python:3.8 +ARG IMAGE_ARCH +FROM public.ecr.aws/lambda/python:3.12-$IMAGE_ARCH WORKDIR /var/task COPY ./ ./ From ba56ed44080dbd27872fb6bbebc9b9197307f163 Mon Sep 17 00:00:00 2001 From: Marco Cieno Date: Tue, 30 Apr 2024 00:13:20 +0200 Subject: [PATCH 04/12] feat: allow user-defined client context (#110) --- cmd/aws-lambda-rie/handlers.go | 9 ++++++ .../local_lambda/test_end_to_end.py | 30 ++++++++++++++++--- test/integration/testdata/main.py | 4 +++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/aws-lambda-rie/handlers.go b/cmd/aws-lambda-rie/handlers.go index 42032cf..2cca12d 100644 --- a/cmd/aws-lambda-rie/handlers.go +++ b/cmd/aws-lambda-rie/handlers.go @@ -5,6 +5,7 @@ package main import ( "bytes" + "encoding/base64" "fmt" "io/ioutil" "math" @@ -81,6 +82,13 @@ func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox, bs i return } + rawClientContext, err := base64.StdEncoding.DecodeString(r.Header.Get("X-Amz-Client-Context")) + if err != nil { + log.Errorf("Failed to decode X-Amz-Client-Context: %s", err) + w.WriteHeader(500) + return + } + initDuration := "" inv := GetenvWithDefault("AWS_LAMBDA_FUNCTION_TIMEOUT", "300") timeoutDuration, _ := time.ParseDuration(inv + "s") @@ -114,6 +122,7 @@ func InvokeHandler(w http.ResponseWriter, r *http.Request, sandbox Sandbox, bs i TraceID: r.Header.Get("X-Amzn-Trace-Id"), LambdaSegmentID: r.Header.Get("X-Amzn-Segment-Id"), Payload: bytes.NewReader(bodyBytes), + ClientContext: string(rawClientContext), } fmt.Println("START RequestId: " + invokePayload.ID + " Version: " + functionVersion) diff --git a/test/integration/local_lambda/test_end_to_end.py b/test/integration/local_lambda/test_end_to_end.py index 7c5486f..8e34b77 100644 --- a/test/integration/local_lambda/test_end_to_end.py +++ b/test/integration/local_lambda/test_end_to_end.py @@ -4,6 +4,8 @@ from subprocess import Popen, PIPE from unittest import TestCase, main from pathlib import Path +import base64 +import json import time import os import requests @@ -62,12 +64,14 @@ def run_command(self, cmd): def sleep_1s(self): time.sleep(SLEEP_TIME) - - def invoke_function(self): + + def invoke_function(self, json={}, headers={}): return requests.post( - f"http://localhost:{self.PORT}/2015-03-31/functions/function/invocations", json={} + f"http://localhost:{self.PORT}/2015-03-31/functions/function/invocations", + json=json, + headers=headers, ) - + @contextmanager def create_container(self, param, image): try: @@ -234,6 +238,24 @@ def test_port_override(self): self.assertEqual(b'"My lambda ran succesfully"', r.content) + def test_custom_client_context(self): + image, rie, image_name = self.tagged_name("custom_client_context") + + params = f"--name {image} -d -v {self.path_to_binary}:/local-lambda-runtime-server -p {self.PORT}:8080 --entrypoint /local-lambda-runtime-server/{rie} {image_name} {DEFAULT_1P_ENTRYPOINT} main.custom_client_context_handler" + + with self.create_container(params, image): + r = self.invoke_function(headers={ + "X-Amz-Client-Context": base64.b64encode(json.dumps({ + "custom": { + "foo": "bar", + "baz": 123, + } + }).encode('utf8')).decode('utf8'), + }) + content = json.loads(r.content) + self.assertEqual("bar", content["foo"]) + self.assertEqual(123, content["baz"]) + if __name__ == "__main__": main() diff --git a/test/integration/testdata/main.py b/test/integration/testdata/main.py index b6b527d..9757be8 100644 --- a/test/integration/testdata/main.py +++ b/test/integration/testdata/main.py @@ -41,3 +41,7 @@ def check_remaining_time_handler(event, context): # Wait 1s to see if the remaining time changes time.sleep(1) return context.get_remaining_time_in_millis() + + +def custom_client_context_handler(event, context): + return context.client_context.custom From d37e08c13600eae4deb1329603f01a78363f360e Mon Sep 17 00:00:00 2001 From: seshubaws <116689586+seshubaws@users.noreply.github.com> Date: Tue, 14 May 2024 11:45:41 -0700 Subject: [PATCH 05/12] Added workflow for automated releases (#121) * Added release workflow --- .github/workflows/release.yml | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..32e878d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +name: Release + +on: + workflow_dispatch: + inputs: + releaseVersion: + description: "Version to use for the release." + required: true + default: "X.Y" + releaseBody: + description: "Information about the release" + required: true + default: "New release" +jobs: + Release: + environment: Release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: main + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build + run: make compile-with-docker-all + - name: Run Integ Tests + run: | + make tests-with-docker + make integ-tests + - name: Release + uses: softprops/action-gh-release@v2 + with: + name: Release ${{ github.event.inputs.releaseVersion }} + tag_name: v${{ github.event.inputs.releaseVersion }} + body: ${{ github.event.inputs.releaseBody }} + files: | + bin/aws-lambda-rie + bin/aws-lambda-rie-arm64 + bin/aws-lambda-rie-x86_64 From 9e6041b151436647af596aec7b48c88f15ac360a Mon Sep 17 00:00:00 2001 From: Renato Valenzuela <37676028+valerena@users.noreply.github.com> Date: Mon, 3 Jun 2024 16:23:06 -0700 Subject: [PATCH 06/12] feat: Add automatic vulnerabilities check (#123) * Add automatic vulnerabilities check --- .github/workflows/check-binaries.yml | 78 ++++++++++++++++++++++++++++ Makefile | 5 +- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/check-binaries.yml diff --git a/.github/workflows/check-binaries.yml b/.github/workflows/check-binaries.yml new file mode 100644 index 0000000..bd41ece --- /dev/null +++ b/.github/workflows/check-binaries.yml @@ -0,0 +1,78 @@ +name: Check binaries + +on: + workflow_dispatch: + schedule: + - cron: "0 16 * * 1-5" # min h d Mo DoW / 9am PST M-F + +jobs: + check-for-vulnerabilities: + runs-on: ubuntu-latest + outputs: + report_contents: ${{ steps.save-output.outputs.report_contents }} + steps: + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: main + - name: Download latest release + uses: robinraju/release-downloader@v1.10 + with: + latest: true + fileName: 'aws-lambda-rie*' + out-file-path: "bin" + - name: Run check for vulnerabilities + id: check-binaries + run: | + make check-binaries + - if: always() && failure() # `always()` to run even if the previous step failed. Failure means that there are vulnerabilities + name: Save content of the vulnerabilities report as GitHub output + id: save-output + run: | + report_csv="$(ls -tr output.cve-bin-*.csv 2>/dev/null | tail -n1)" # last file generated + echo "Vulnerabilities stored in $report_csv" + final_report="${report_csv}.txt" + awk -F',' '{n=split($10, path, "/"); print $2,$3,$4,$5,path[n]}' "$report_csv" | column -t > "$final_report" # make the CSV nicer + echo "report_contents<> "$GITHUB_OUTPUT" + cat "$final_report" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + - if: always() && steps.check-binaries.outcome == 'failure' + name: Build new binaries and check vulnerabilities again + id: check-new-version + run: | + mkdir ./bin2 + mv ./bin/* ./bin2 + make compile-with-docker-all + latest_version=$(strings bin/aws-lambda-rie* | grep '^go1\.' | sort | uniq) + echo "latest_version=$latest_version" >> "$GITHUB_OUTPUT" + make check-binaries + - if: always() && steps.check-binaries.outcome == 'failure' + name: Save outputs for the check with the latest build + id: save-new-version + run: | + if [ "${{ steps.check-new-version.outcome }}" == "failure" ]; then + fixed="No" + else + fixed="Yes" + fi + echo "fixed=$fixed" >> "$GITHUB_OUTPUT" + - if: always() && steps.check-binaries.outcome == 'failure' + name: Create GitHub Issue indicating vulnerabilities + id: create-issue + uses: dacbd/create-issue-action@main + with: + token: ${{ github.token }} + title: | + CVEs found in latest RIE release + body: | + ### CVEs found in latest RIE release + ``` + ${{ steps.save-output.outputs.report_contents }} + ``` + + #### Are these resolved by building with the latest patch version of Go (${{ steps.check-new-version.outputs.latest_version }})?: + > **${{ steps.save-new-version.outputs.fixed }}** diff --git a/Makefile b/Makefile index f7a714e..6b66e79 100644 --- a/Makefile +++ b/Makefile @@ -70,4 +70,7 @@ integ-tests-with-docker-old: make ARCH=old compile-with-docker make prep-python make TEST_ARCH="" TEST_PORT=9052 exec-python-e2e-test - \ No newline at end of file + +check-binaries: prep-python + .venv/bin/pip install cve-bin-tool + .venv/bin/python -m cve_bin_tool.cli bin/ -r go -d REDHAT,OSV,GAD,CURL --no-0-cve-report -f csv From 71388dd788b7a5519262391ce73fe6548dbaf86e Mon Sep 17 00:00:00 2001 From: Renato Valenzuela <37676028+valerena@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:51:08 -0700 Subject: [PATCH 07/12] fix: Vulnerability checks: create issue only when checked was done (#125) --- .github/workflows/check-binaries.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-binaries.yml b/.github/workflows/check-binaries.yml index bd41ece..75fa28f 100644 --- a/.github/workflows/check-binaries.yml +++ b/.github/workflows/check-binaries.yml @@ -34,13 +34,17 @@ jobs: id: save-output run: | report_csv="$(ls -tr output.cve-bin-*.csv 2>/dev/null | tail -n1)" # last file generated - echo "Vulnerabilities stored in $report_csv" + if [ -z "$report_csv" ]; then + echo "No file with vulnerabilities. Probably a failure in previous step." + else + echo "Vulnerabilities stored in $report_csv" + fi final_report="${report_csv}.txt" awk -F',' '{n=split($10, path, "/"); print $2,$3,$4,$5,path[n]}' "$report_csv" | column -t > "$final_report" # make the CSV nicer echo "report_contents<> "$GITHUB_OUTPUT" cat "$final_report" >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" - - if: always() && steps.check-binaries.outcome == 'failure' + - if: always() && steps.save-output.outputs.report_contents name: Build new binaries and check vulnerabilities again id: check-new-version run: | @@ -50,7 +54,7 @@ jobs: latest_version=$(strings bin/aws-lambda-rie* | grep '^go1\.' | sort | uniq) echo "latest_version=$latest_version" >> "$GITHUB_OUTPUT" make check-binaries - - if: always() && steps.check-binaries.outcome == 'failure' + - if: always() && steps.save-output.outputs.report_contents name: Save outputs for the check with the latest build id: save-new-version run: | @@ -60,7 +64,7 @@ jobs: fixed="Yes" fi echo "fixed=$fixed" >> "$GITHUB_OUTPUT" - - if: always() && steps.check-binaries.outcome == 'failure' + - if: always() && steps.save-output.outputs.report_contents name: Create GitHub Issue indicating vulnerabilities id: create-issue uses: dacbd/create-issue-action@main From 781cd9a296b10ed44a0223b34092703ddc4a36b6 Mon Sep 17 00:00:00 2001 From: Renato Valenzuela <37676028+valerena@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:16:03 -0800 Subject: [PATCH 08/12] test: Add delay for time-related arm64 tests (#138) The tests on GitHub run on x86 instances (because arm64 instances don't have Docker installed) so when running the arm64 tests, invokes take longer because of the cross-architecture emulation. This caused that some tests that check remaining time in the function were not landing on the correct time range. It's unknown why this delay started manifesting more consistently, we might want to find a better solution in the future. --- .../local_lambda/test_end_to_end.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/integration/local_lambda/test_end_to_end.py b/test/integration/local_lambda/test_end_to_end.py index 8e34b77..d564bb1 100644 --- a/test/integration/local_lambda/test_end_to_end.py +++ b/test/integration/local_lambda/test_end_to_end.py @@ -173,9 +173,8 @@ def test_context_get_remaining_time_in_three_seconds(self): with self.create_container(params, image): r = self.invoke_function() - # Execution time is not decided, 1.0s ~ 3.0s is a good estimation - self.assertLess(int(r.content), 3000) - self.assertGreater(int(r.content), 1000) + # Execution time is not decided, but it should be around 2.0s + self.assertAround(int(r.content), 2000) def test_context_get_remaining_time_in_ten_seconds(self): @@ -186,9 +185,8 @@ def test_context_get_remaining_time_in_ten_seconds(self): with self.create_container(params, image): r = self.invoke_function() - # Execution time is not decided, 8.0s ~ 10.0s is a good estimation - self.assertLess(int(r.content), 10000) - self.assertGreater(int(r.content), 8000) + # Execution time is not decided, but it should be around 9.0s + self.assertAround(int(r.content), 9000) def test_context_get_remaining_time_in_default_deadline(self): @@ -199,9 +197,8 @@ def test_context_get_remaining_time_in_default_deadline(self): with self.create_container(params, image): r = self.invoke_function() - # Executation time is not decided, 298.0s ~ 300.0s is a good estimation - self.assertLess(int(r.content), 300000) - self.assertGreater(int(r.content), 298000) + # Execution time is not decided, but it should be around 299.0s + self.assertAround(int(r.content), 299000) def test_invoke_with_pre_runtime_api_runtime(self): @@ -256,6 +253,13 @@ def test_custom_client_context(self): self.assertEqual("bar", content["foo"]) self.assertEqual(123, content["baz"]) + def assertAround(self, number, target): + # Emulating arm64 on x86 causes the invoke to take longer + delay_arm64 = 500 + actual_target = target if self.ARCH != 'arm64' else target - delay_arm64 + + self.assertLess(number, actual_target + 1000) + self.assertGreater(number, actual_target - 1000) if __name__ == "__main__": main() From 3a0772eae98d7653006b259e6be9c2a8e5b32d88 Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Wed, 23 Apr 2025 10:31:40 -0700 Subject: [PATCH 09/12] chore(deps): Update to Go 1.24 (#143) * update to 1.24 * fix changes --- Makefile | 2 +- go.mod | 2 +- lambda/rapi/handler/agentnext.go | 2 +- lambda/rapi/handler/agentregister.go | 2 +- lambda/rapi/handler/runtimelogs.go | 4 ++-- lambda/rapid/handlers.go | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6b66e79..077cf31 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ DESTINATION_old:= bin/${BINARY_NAME} DESTINATION_x86_64 := bin/${BINARY_NAME}-x86_64 DESTINATION_arm64 := bin/${BINARY_NAME}-arm64 -run_in_docker = docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.22 $(1) +run_in_docker = docker run --env GOPROXY=direct -v $(shell pwd):/LambdaRuntimeLocal -w /LambdaRuntimeLocal golang:1.24 $(1) compile-with-docker-all: $(call run_in_docker, make compile-lambda-linux-all) diff --git a/go.mod b/go.mod index 4ee45d7..5519b8c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.amzn.com -go 1.22 +go 1.24 require ( github.com/aws/aws-lambda-go v1.46.0 diff --git a/lambda/rapi/handler/agentnext.go b/lambda/rapi/handler/agentnext.go index 7ce76f0..ffdd61d 100644 --- a/lambda/rapi/handler/agentnext.go +++ b/lambda/rapi/handler/agentnext.go @@ -48,7 +48,7 @@ func (h *agentNextHandler) ServeHTTP(writer http.ResponseWriter, request *http.R } } else { log.Warnf("Unknown agent %s tried to call /next", agentID.String()) - rendering.RenderForbiddenWithTypeMsg(writer, request, errAgentIdentifierUnknown, "Unknown extension"+agentID.String()) + rendering.RenderForbiddenWithTypeMsg(writer, request, errAgentIdentifierUnknown, "Unknown extension %s", agentID.String()) return } diff --git a/lambda/rapi/handler/agentregister.go b/lambda/rapi/handler/agentregister.go index 8da9e4c..867ad9d 100644 --- a/lambda/rapi/handler/agentregister.go +++ b/lambda/rapi/handler/agentregister.go @@ -77,7 +77,7 @@ func (h *agentRegisterHandler) ServeHTTP(writer http.ResponseWriter, request *ht registerRequest, err := parseRegister(request) if err != nil { - rendering.RenderForbiddenWithTypeMsg(writer, request, errInvalidRequestFormat, err.Error()) + rendering.RenderForbiddenWithTypeMsg(writer, request, errInvalidRequestFormat, "%s", err.Error()) return } diff --git a/lambda/rapi/handler/runtimelogs.go b/lambda/rapi/handler/runtimelogs.go index 6b8a67e..4fd534e 100644 --- a/lambda/rapi/handler/runtimelogs.go +++ b/lambda/rapi/handler/runtimelogs.go @@ -30,7 +30,7 @@ func (h *runtimeLogsHandler) ServeHTTP(writer http.ResponseWriter, request *http log.Errorf("Agent Verification Error: %s", err) switch err := err.(type) { case *ErrAgentIdentifierUnknown: - rendering.RenderForbiddenWithTypeMsg(writer, request, errAgentIdentifierUnknown, "Unknown extension "+err.agentID.String()) + rendering.RenderForbiddenWithTypeMsg(writer, request, errAgentIdentifierUnknown, "Unknown extension %s", err.agentID.String()) h.telemetrySubscription.RecordCounterMetric(telemetry.SubscribeClientErr, 1) default: rendering.RenderInternalServerError(writer, request) @@ -55,7 +55,7 @@ func (h *runtimeLogsHandler) ServeHTTP(writer http.ResponseWriter, request *http switch err { case telemetry.ErrTelemetryServiceOff: rendering.RenderForbiddenWithTypeMsg(writer, request, - h.telemetrySubscription.GetServiceClosedErrorType(), h.telemetrySubscription.GetServiceClosedErrorMessage()) + h.telemetrySubscription.GetServiceClosedErrorType(), "%s", h.telemetrySubscription.GetServiceClosedErrorMessage()) h.telemetrySubscription.RecordCounterMetric(telemetry.SubscribeClientErr, 1) default: rendering.RenderInternalServerError(writer, request) diff --git a/lambda/rapid/handlers.go b/lambda/rapid/handlers.go index f379c4c..2e759e9 100644 --- a/lambda/rapid/handlers.go +++ b/lambda/rapid/handlers.go @@ -243,7 +243,7 @@ func (c *rapidContext) watchEvents(events <-chan supvmodel.Event) { if termination.Success() { err = fmt.Errorf("exit code 0") } else { - err = fmt.Errorf(termination.String()) + err = fmt.Errorf("%s", termination.String()) } appctx.StoreFirstFatalError(c.appCtx, fatalerror.AgentCrash) @@ -851,7 +851,7 @@ func handleRestore(execCtx *rapidContext, restore *interop.Restore) (interop.Res // check if there is any error stored in appctx to get the root cause error type // Runtime.ExitError is an example to such a scenario if fatalErrorFound { - err = fmt.Errorf(string(fatalErrorType)) + err = fmt.Errorf("%s", string(fatalErrorType)) } if err != nil { From 85e53022a36cf95bc7f06146fa53617eb9d652f6 Mon Sep 17 00:00:00 2001 From: Frederic Mbea <117131783+mbfreder@users.noreply.github.com> Date: Fri, 8 Aug 2025 14:45:54 -0700 Subject: [PATCH 10/12] chore(deps): Update go-chi to v5.2.2 (#149) * chore(deps): Update go-chi to v5.2.2 --- go.mod | 2 +- go.sum | 4 ++-- lambda/core/directinvoke/directinvoke.go | 2 +- lambda/core/directinvoke/directinvoke_test.go | 2 +- lambda/rapi/handler/invocationerror.go | 2 +- lambda/rapi/handler/invocationerror_test.go | 2 +- lambda/rapi/handler/invocationresponse.go | 2 +- lambda/rapi/middleware/middleware.go | 2 +- lambda/rapi/middleware/middleware_test.go | 2 +- lambda/rapi/router.go | 2 +- lambda/rapi/server.go | 2 +- lambda/rapidcore/standalone/middleware.go | 2 +- lambda/rapidcore/standalone/router.go | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 5519b8c..5118f72 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.24 require ( github.com/aws/aws-lambda-go v1.46.0 - github.com/go-chi/chi v1.5.5 + github.com/go-chi/chi/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 8974775..9fd0886 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,8 @@ github.com/aws/aws-lambda-go v1.46.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7Rfg github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= -github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= +github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= +github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= diff --git a/lambda/core/directinvoke/directinvoke.go b/lambda/core/directinvoke/directinvoke.go index 3510132..6b97cbe 100644 --- a/lambda/core/directinvoke/directinvoke.go +++ b/lambda/core/directinvoke/directinvoke.go @@ -11,7 +11,7 @@ import ( "strconv" "strings" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "go.amzn.com/lambda/core/bandwidthlimiter" "go.amzn.com/lambda/fatalerror" "go.amzn.com/lambda/interop" diff --git a/lambda/core/directinvoke/directinvoke_test.go b/lambda/core/directinvoke/directinvoke_test.go index 94b6323..e6646bf 100644 --- a/lambda/core/directinvoke/directinvoke_test.go +++ b/lambda/core/directinvoke/directinvoke_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.amzn.com/lambda/fatalerror" diff --git a/lambda/rapi/handler/invocationerror.go b/lambda/rapi/handler/invocationerror.go index d434461..afd346d 100644 --- a/lambda/rapi/handler/invocationerror.go +++ b/lambda/rapi/handler/invocationerror.go @@ -18,7 +18,7 @@ import ( "go.amzn.com/lambda/core" "go.amzn.com/lambda/rapi/rendering" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" log "github.com/sirupsen/logrus" ) diff --git a/lambda/rapi/handler/invocationerror_test.go b/lambda/rapi/handler/invocationerror_test.go index 72e6719..76a3b4e 100644 --- a/lambda/rapi/handler/invocationerror_test.go +++ b/lambda/rapi/handler/invocationerror_test.go @@ -19,7 +19,7 @@ import ( "go.amzn.com/lambda/rapi/model" "go.amzn.com/lambda/testdata" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "github.com/stretchr/testify/assert" ) diff --git a/lambda/rapi/handler/invocationresponse.go b/lambda/rapi/handler/invocationresponse.go index d267775..ba08b14 100644 --- a/lambda/rapi/handler/invocationresponse.go +++ b/lambda/rapi/handler/invocationresponse.go @@ -12,7 +12,7 @@ import ( "go.amzn.com/lambda/interop" "go.amzn.com/lambda/rapi/rendering" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" log "github.com/sirupsen/logrus" ) diff --git a/lambda/rapi/middleware/middleware.go b/lambda/rapi/middleware/middleware.go index d45798b..a83a232 100644 --- a/lambda/rapi/middleware/middleware.go +++ b/lambda/rapi/middleware/middleware.go @@ -12,7 +12,7 @@ import ( "go.amzn.com/lambda/rapi/handler" "go.amzn.com/lambda/rapi/rendering" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "go.amzn.com/lambda/appctx" log "github.com/sirupsen/logrus" diff --git a/lambda/rapi/middleware/middleware_test.go b/lambda/rapi/middleware/middleware_test.go index a0b9134..51c9a90 100644 --- a/lambda/rapi/middleware/middleware_test.go +++ b/lambda/rapi/middleware/middleware_test.go @@ -12,7 +12,7 @@ import ( "net/http/httptest" "testing" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "github.com/google/uuid" "github.com/stretchr/testify/assert" "go.amzn.com/lambda/appctx" diff --git a/lambda/rapi/router.go b/lambda/rapi/router.go index dc036bc..9933550 100644 --- a/lambda/rapi/router.go +++ b/lambda/rapi/router.go @@ -11,7 +11,7 @@ import ( "go.amzn.com/lambda/rapi/middleware" "go.amzn.com/lambda/telemetry" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "go.amzn.com/lambda/core" "go.amzn.com/lambda/rapi/rendering" diff --git a/lambda/rapi/server.go b/lambda/rapi/server.go index d17270a..aafc295 100644 --- a/lambda/rapi/server.go +++ b/lambda/rapi/server.go @@ -9,7 +9,7 @@ import ( "net" "net/http" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" "go.amzn.com/lambda/appctx" "go.amzn.com/lambda/core" diff --git a/lambda/rapidcore/standalone/middleware.go b/lambda/rapidcore/standalone/middleware.go index 06baae3..4465a40 100644 --- a/lambda/rapidcore/standalone/middleware.go +++ b/lambda/rapidcore/standalone/middleware.go @@ -6,7 +6,7 @@ package standalone import ( "net/http" - "github.com/go-chi/chi/middleware" + "github.com/go-chi/chi/v5/middleware" log "github.com/sirupsen/logrus" ) diff --git a/lambda/rapidcore/standalone/router.go b/lambda/rapidcore/standalone/router.go index 7957c32..c0b8fc1 100644 --- a/lambda/rapidcore/standalone/router.go +++ b/lambda/rapidcore/standalone/router.go @@ -12,7 +12,7 @@ import ( "go.amzn.com/lambda/rapidcore" "go.amzn.com/lambda/rapidcore/standalone/telemetry" - "github.com/go-chi/chi" + "github.com/go-chi/chi/v5" ) type InteropServer interface { From e5440b64bd74cefd3a70aaff9c7615343ea700f5 Mon Sep 17 00:00:00 2001 From: Frederic Mbea <117131783+mbfreder@users.noreply.github.com> Date: Mon, 11 Aug 2025 15:37:02 -0700 Subject: [PATCH 11/12] Update workflow to run integ-tests in main (#151) --- .github/workflows/integ-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integ-tests.yml b/.github/workflows/integ-tests.yml index 7fddc95..92be22e 100644 --- a/.github/workflows/integ-tests.yml +++ b/.github/workflows/integ-tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - develop + - main jobs: go-tests: From c17162c6dd23541f71e7cf529868f51f397e309e Mon Sep 17 00:00:00 2001 From: Roger Zhang Date: Tue, 16 Sep 2025 15:06:58 -0700 Subject: [PATCH 12/12] Update README.md to make it clear RIE binary is meant to be used in linux env (#152) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3599cc0..3a37485 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,16 @@ Lambda’s orchestrator, or security and authentication configurations. You can ## Installing -Instructions for installing AWS Lambda Runtime Interface Emulator for your platform +The following commands download the RIE binary for your platform. Note that while you can download the binary on any platform, the RIE can only be executed in a Linux environment (typically within a Docker container). -| Platform | Command to install | +| Platform (for downloading) | Command to download | |---------|--------- | macOS/Linux x86\_64 | `mkdir -p ~/.aws-lambda-rie && curl -Lo ~/.aws-lambda-rie/aws-lambda-rie https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie && chmod +x ~/.aws-lambda-rie/aws-lambda-rie` | | macOS/Linux arm64 | `mkdir -p ~/.aws-lambda-rie && curl -Lo ~/.aws-lambda-rie/aws-lambda-rie https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-arm64 && chmod +x ~/.aws-lambda-rie/aws-lambda-rie` | | Windows x86\_64 | `Invoke-WebRequest -OutFile 'C:\Program Files\aws lambda\aws-lambda-rie' https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie` | | Windows arm64 | `Invoke-WebRequest -OutFile 'C:\Program Files\aws lambda\aws-lambda-rie' https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie-arm64` | +After downloading, the RIE binary must be used within a Linux environment, typically as part of a Docker container setup. See the Docker configuration instructions below for proper implementation. ## Getting started