diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml new file mode 100644 index 0000000..17c963d --- /dev/null +++ b/.github/workflows/spelling.yml @@ -0,0 +1,239 @@ +name: Check Spelling + +# Comment management is handled through a secondary job, for details see: +# https://github.com/check-spelling/check-spelling/wiki/Feature%3A-Restricted-Permissions +# +# `jobs.comment-push` runs when a push is made to a repository and the `jobs.spelling` job needs to make a comment +# (in odd cases, it might actually run just to collapse a comment, but that's fairly rare) +# it needs `contents: write` in order to add a comment. +# +# `jobs.comment-pr` runs when a pull_request is made to a repository and the `jobs.spelling` job needs to make a comment +# or collapse a comment (in the case where it had previously made a comment and now no longer needs to show a comment) +# it needs `pull-requests: write` in order to manipulate those comments. + +# Updating pull request branches is managed via comment handling. +# For details, see: https://github.com/check-spelling/check-spelling/wiki/Feature:-Update-expect-list +# +# These elements work together to make it happen: +# +# `on.issue_comment` +# This event listens to comments by users asking to update the metadata. +# +# `jobs.update` +# This job runs in response to an issue_comment and will push a new commit +# to update the spelling metadata. +# +# `with.experimental_apply_changes_via_bot` +# Tells the action to support and generate messages that enable it +# to make a commit to update the spelling metadata. +# +# `with.ssh_key` +# In order to trigger workflows when the commit is made, you can provide a +# secret (typically, a write-enabled github deploy key). +# +# For background, see: https://github.com/check-spelling/check-spelling/wiki/Feature:-Update-with-deploy-key + +# SARIF reporting +# +# Access to SARIF reports is generally restricted (by GitHub) to members of the repository. +# +# Requires enabling `security-events: write` +# and configuring the action with `use_sarif: 1` +# +# For information on the feature, see: https://github.com/check-spelling/check-spelling/wiki/Feature:-SARIF-output + +# Minimal workflow structure: +# +# on: +# push: +# ... +# pull_request_target: +# ... +# jobs: +# # you only want the spelling job, all others should be omitted +# spelling: +# # remove `security-events: write` and `use_sarif: 1` +# # remove `experimental_apply_changes_via_bot: 1` +# ... otherwise adjust the `with:` as you wish + +on: + push: + branches: + - '**' + tags-ignore: + - '**' + pull_request_target: + branches: + - '**' + types: + - 'opened' + - 'reopened' + - 'synchronize' + issue_comment: + types: + - 'created' + +jobs: + spelling: + name: Check Spelling + permissions: + contents: read + pull-requests: read + actions: read + security-events: write + outputs: + followup: ${{ steps.spelling.outputs.followup }} + runs-on: ubuntu-latest + if: ${{ contains(github.event_name, 'pull_request') || github.event_name == 'push' }} + concurrency: + group: spelling-${{ github.event.pull_request.number || github.ref }} + # note: If you use only_check_changed_files, you do not want cancel-in-progress + cancel-in-progress: true + steps: + - name: check-spelling + id: spelling + uses: check-spelling/check-spelling@prerelease + with: + suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} + checkout: true + check_file_names: 1 + spell_check_this: check-spelling/spell-check-this@prerelease + post_comment: 0 + use_magic_file: 1 + report-timing: 1 + warnings: bad-regex,binary-file,deprecated-feature,ignored-expect-variant,large-file,limited-references,no-newline-at-eof,noisy-file,non-alpha-in-dictionary,token-is-substring,unexpected-line-ending,whitespace-in-dictionary,minified-file,unsupported-configuration,no-files-to-check,unclosed-block-ignore-begin,unclosed-block-ignore-end + experimental_apply_changes_via_bot: 1 + use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }} + extra_dictionary_limit: 20 + dictionary_source_prefixes: > + { + "cspell": "https://raw.githubusercontent.com/check-spelling/cspell-dicts/v20230509/dictionaries/", + "cspell1": "https://raw.githubusercontent.com/check-spelling/cspell-dicts/v20241114/dictionaries/" + } + check_extra_dictionaries: | + cspell1:ada/ada.txt + cspell1:aws/aws.txt + cspell1:clojure/clojure.txt + cspell1:companies/companies.txt + cspell1:cpp/compiler-clang-attributes.txt + cspell1:cpp/compiler-gcc.txt + cspell1:cpp/compiler-msvc.txt + cspell1:cpp/ecosystem.txt + cspell1:cpp/lang-jargon.txt + cspell1:cpp/lang-keywords.txt + cspell1:cpp/people.txt + cspell1:cpp/stdlib-c.txt + cspell1:cpp/stdlib-cerrno.txt + cspell1:cpp/stdlib-cmath.txt + cspell1:cpp/stdlib-cpp.txt + cspell1:cpp/template-strings.txt + cspell1:cryptocurrencies/cryptocurrencies.txt + cspell1:csharp/csharp.txt + cspell1:css/css.txt + cspell1:dart/dart.txt + cspell1:django/django.txt + cspell1:django/requirements.txt + cspell1:docker/docker-words.txt + cspell1:dotnet/dotnet.txt + cspell1:elixir/elixir.txt + cspell1:filetypes/filetypes.txt + cspell1:fonts/fonts.txt + cspell1:fullstack/fullstack.txt + cspell1:gaming-terms/gaming-terms.txt + cspell1:golang/go.txt + cspell1:haskell/haskell.txt + cspell1:html/html.txt + cspell1:java/java-terms.txt + cspell1:java/java.txt + cspell1:k8s/k8s.txt + cspell1:latex/latex.txt + cspell1:latex/samples/sample-words.txt + cspell1:lisp/lisp.txt + cspell1:lorem-ipsum/dictionary.txt + cspell1:lua/lua.txt + cspell1:mnemonics/mnemonics.txt + cspell1:monkeyc/monkeyc_keywords.txt + cspell1:node/node.txt + cspell1:npm/npm.txt + cspell1:php/php.txt + cspell1:powershell/powershell.txt + cspell1:public-licenses/generated/public-licenses.txt + cspell1:python/additional_words.txt + cspell1:python/common/extra.txt + cspell1:python/python/python-lib.txt + cspell1:python/python/python.txt + cspell1:r/r.txt + cspell1:redis/redis.txt + cspell1:ruby/ruby.txt + cspell1:rust/rust.txt + cspell1:scala/scala.txt + cspell1:shell/shell-all-words.txt + cspell1:software-terms/software-terms.txt + cspell1:software-terms/webServices.txt + cspell1:sql/sql.txt + cspell1:sql/tsql.txt + cspell1:svelte/svelte.txt + cspell1:swift/swift.txt + cspell1:typescript/typescript.txt + extra_dictionaries: | + cspell1:software-terms/softwareTerms.txt + + comment-push: + name: Report (Push) + # If your workflow isn't running on push, you can remove this job + runs-on: ubuntu-latest + needs: spelling + permissions: + actions: read + contents: write + if: (success() || failure()) && needs.spelling.outputs.followup && github.event_name == 'push' + steps: + - name: comment + uses: check-spelling/check-spelling@prerelease + with: + checkout: true + spell_check_this: check-spelling/spell-check-this@prerelease + task: ${{ needs.spelling.outputs.followup }} + + comment-pr: + name: Report (PR) + # If you workflow isn't running on pull_request*, you can remove this job + runs-on: ubuntu-latest + needs: spelling + permissions: + actions: read + contents: read + pull-requests: write + if: (success() || failure()) && needs.spelling.outputs.followup && contains(github.event_name, 'pull_request') + steps: + - name: comment + uses: check-spelling/check-spelling@prerelease + with: + checkout: true + spell_check_this: check-spelling/spell-check-this@prerelease + task: ${{ needs.spelling.outputs.followup }} + experimental_apply_changes_via_bot: 1 + + update: + name: Update PR + permissions: + contents: write + pull-requests: write + actions: read + runs-on: ubuntu-latest + if: ${{ + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + contains(github.event.comment.body, '@check-spelling-bot apply') && + contains(github.event.comment.body, 'https://') + }} + concurrency: + group: spelling-update-${{ github.event.issue.number }} + cancel-in-progress: false + steps: + - name: apply spelling updates + uses: check-spelling/check-spelling@prerelease + with: + experimental_apply_changes_via_bot: 1 + checkout: true + ssh_key: '${{ secrets.CHECK_SPELLING }}' diff --git a/ECSLogo.zip b/ECSLogo.zip deleted file mode 100644 index eb73fb7..0000000 Binary files a/ECSLogo.zip and /dev/null differ diff --git a/ECSLogo/AWS_Logo_PoweredBy_300px.png b/ECSLogo/AWS_Logo_PoweredBy_300px.png deleted file mode 100644 index da4f232..0000000 Binary files a/ECSLogo/AWS_Logo_PoweredBy_300px.png and /dev/null differ diff --git a/ECSLogo/ECSLogo.ini b/ECSLogo/ECSLogo.ini deleted file mode 100644 index a7eb9f3..0000000 --- a/ECSLogo/ECSLogo.ini +++ /dev/null @@ -1,20 +0,0 @@ -; Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -; -; Licensed under the Apache License, Version 2.0 (the "License"). -; You may not use this file except in compliance with the License. -; A copy of the License is located at -; -; http://aws.amazon.com/apache2.0/ -; -; or in the "license" file accompanying this file. -; This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -; See the License for the specific language governing permissions and limitations under the License. - -; Options for rendering ECSLogo.pov -Input_File_Name=ECSLogo.pov -+W1920 +H1080 -Quality=11 -Antialias=on -Sampling_Method=2 -Antialias_Depth=5 -Jitter=on diff --git a/ECSLogo/ECSLogo.pov b/ECSLogo/ECSLogo.pov deleted file mode 100644 index 895ee7e..0000000 --- a/ECSLogo/ECSLogo.pov +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). -// You may not use this file except in compliance with the License. -// A copy of the License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. -// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and limitations under the License. - -// The Amazon EC2 Container Service (ECS) Logo, powered by AWS. - -// Includes. -#include "colors.inc" -#include "rad_def.inc" -#include "finish.inc" - -// Global settings and defaults. -global_settings { - radiosity { - Rad_Settings(Radiosity_OutdoorHQ,off,off) - } -} -#default {finish{ambient 0}} - -// Declarations. -#declare ImageWidth = 1920; -#declare ImageHeight = 1080; - -#declare AWSOrange = rgb <1, .5, 0>; - -#declare PoweredByAWSLogoImageFile = "AWS_Logo_PoweredBy_300px.png" -#declare PoweredByAWSLogoImageWidth = 300; -#declare PoweredByAWSLogoImageHeight = 122; -#declare PoweredByAWSLogoAspect = PoweredByAWSLogoImageHeight / PoweredByAWSLogoImageWidth; - -#declare PoweredByAWSLogoSize = 0.5; -#declare PoweredByAWSLogoDepth = .00001; -#declare PoweredByAWSLogoXPosition = 1.3; -#declare PoweredByAWSLogoYPosition = 0.8; - -#declare ECSPlaneThickness = 0.1; -#declare ECSPlaneSpacing = ECSPlaneThickness * 2; -#declare ECSFrameInset = 3/5; -#declare ECSContainerSpacing = ECSFrameInset / 3; -#declare ECSContainerWidth = (ECSFrameInset - ECSContainerSpacing) / 2; -#declare ECSContainerHeight = ECSContainerWidth; -#declare ECSContainerLength = ECSContainerWidth * 2; -#declare ECSContainerXOffset = (ECSContainerWidth + ECSContainerSpacing) / 2; -#declare ECSContainerYOffset = (ECSContainerHeight + ECSContainerSpacing) / 2; -#declare ECSContainerZOffset = ECSContainerLength; - -#declare ECSLogoTexture = texture { - pigment { - color AWSOrange - } -} - -#declare ECSFrame = difference { - box { - <-0.5, 0.5, -ECSPlaneThickness/2>, - < 0.5, -0.5, ECSPlaneThickness/2> - texture { - ECSLogoTexture - } - } - box { - <-ECSFrameInset/2, ECSFrameInset/2, -ECSPlaneThickness>, - < ECSFrameInset/2, -ECSFrameInset/2, ECSPlaneThickness> - } - cutaway_textures -} - -#declare ECSFrameExtra = box { - <-0.5, 0.5, -ECSPlaneThickness/2>, - <-0.5 + ECSFrameInset / 2, -0.5, ECSPlaneThickness/2> - texture { - ECSLogoTexture - } -} - -#declare ECSContainer = box { - <-ECSContainerWidth/2, ECSContainerHeight/2, -ECSContainerLength/2>, - < ECSContainerWidth/2, -ECSContainerHeight/2, ECSContainerLength/2> - texture { - ECSLogoTexture - } -} - -#declare ECSLogo = union { - object { - ECSFrameExtra - translate <0, 0, ECSPlaneSpacing> - } - object { - ECSFrame - } - object { // top left container - ECSContainer - translate <-ECSContainerXOffset, ECSContainerYOffset, -ECSContainerZOffset> - } - object { // top right container - ECSContainer - translate < ECSContainerXOffset, ECSContainerYOffset, -ECSContainerZOffset> - } - object { // bottom left container - ECSContainer - translate <-ECSContainerXOffset, -ECSContainerYOffset, -ECSContainerZOffset> - } - object { // bottom right container - ECSContainer - translate < ECSContainerXOffset, -ECSContainerYOffset, -ECSContainerZOffset> - } -} - -#declare PoweredByAWSLogo = box { - <-0.5, PoweredByAWSLogoAspect / 2, -PoweredByAWSLogoDepth>, - < 0.5, -PoweredByAWSLogoAspect / 2, PoweredByAWSLogoDepth> - texture { - pigment { - color White - } - finish { - ambient 0 - diffuse 1 - } - } - texture { - pigment { - image_map { - png PoweredByAWSLogoImageFile - map_type 0 - } - } - finish { - ambient 0 - diffuse 1 - } - translate <-0.5, -0.5, 0> - scale <1, PoweredByAWSLogoAspect, 1> - } -} - -// The scene. - -background { color White } - -plane { - y, -0.5 - texture { - pigment { - color White - } - } - finish { - Glossy - } -} - -object { - ECSLogo - rotate <0, -45, 0> -} - -object { - PoweredByAWSLogo - scale PoweredByAWSLogoSize - translate -} - -light_source { - < 5, 10, -10> - color White * 0.6 - area_light <5, 0, 0>, <0, 0, 5>, 5, 5 - adaptive 1 - jitter -} - -camera { - right x * ImageWidth/ImageHeight - location <0, 0, -2> - look_at <0, 0, 0> -} diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 5750a7f..0000000 --- a/LICENSE +++ /dev/null @@ -1,40 +0,0 @@ -Apache License -Version 2.0, January 2004 -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -1. Definitions. - -“License” shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -“Licensor” shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -“Legal Entity” shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, “control” means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -“You” (or “Your”) shall mean an individual or Legal Entity exercising permissions granted by this License. - -“Source” form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -“Object” form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -“Work” shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -“Derivative Works” shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -“Contribution” shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, “submitted” means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as “Not a Contribution.” - -“Contributor” shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and -You must cause any modified files to carry prominent notices stating that You changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and -If the Work includes a “NOTICE” text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. -You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. -END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/LambdaECSWorkerPattern.png b/LambdaECSWorkerPattern.png deleted file mode 100644 index 4eb81ef..0000000 Binary files a/LambdaECSWorkerPattern.png and /dev/null differ diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 4667a8d..0000000 --- a/NOTICE +++ /dev/null @@ -1,12 +0,0 @@ -Lambda ECS Worker Pattern Demo -Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may not use this file except in compliance with the License. -A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. -This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 6728e9e..0000000 --- a/README.md +++ /dev/null @@ -1,121 +0,0 @@ -Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may not use this file except in compliance with the License. -A copy of the License is located at - - http://aws.amazon.com/apache2.0/ - -or in the "license" file accompanying this file. -This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and limitations under the License. - -# Lambda ECS Worker Pattern - -This code example illustrates the concept of using Amazon ECS Tasks to extend the functionality of AWS Lambda. - -In this pattern, an AWS Lambda function is triggered by an Amazon S3 event. The AWS Lambda function then pushes the -event data into an Amazon SQS queue and starts an Amazon ECS Task. A simple shell script running inside the Amazon ECS -Task’s container fetches the message from the Amazon SQS queue and processes it. - -You can read more about this pattern in the blog post: [Better Together: Amazon ECS and AWS Lambda](https://aws.amazon.com/blogs/compute/better-together-amazon-ecs-and-aws-lambda/) - -![Architecture overview of the Lambda ECS Worker Pattern](LambdaECSWorkerPattern.png) - -As a demo, we use this pattern to implement a ray-tracing worker using the popular open source -[POV-Ray](http://www.povray.org/) ray-tracer, triggered by uploading a POV-Ray scene description wrapped -into a .ZIP file in an Amazon S3 bucket. Running a ray-tracer inside AWS Lambda would probably take more than the -limit of 60 seconds to complete, so we use Lambda to push the Amazon S3 Notification data into an Amazon SQS queue and -start an Amazon ECS Task for the actual processing. The shell script running in the Amazon ECS Task’s container takes -care of fetching the input data from Amazon S3, running the POV-Ray ray-tracing software and uploading the result image -back into the Amazon S3 bucket. - -## Files - -The following files are included in this repository: - -* README.txt: This file. -* LICENSE.txt: Apache 2.0 license under which this code is licensed. -* NOTICE.txt: Notice about the licensing of this code. -* ECSLogo: A directory containing the POV-Ray source for a demo image. - * AWS_Logo_PoweredBy_300px.png: Official "Powered by AWS" logo image. - * ECSLogo.ini: A POV-Ray .INI file containing rendering parameters. - * ECSLogo.poc: A POV-Ray scene description file that renders the Amazon ECS Logo as a demo. -* ECSLogo.zip: The ZIPped contents of the ECSLogo directory. -* ecs-worker: A directory containing the worker shell script for the Amazon ECS Task. - * ecs-worker.sh : The shell script to be run in a Docker Container as part of the Amazon ECS task. -* ecs-worker-launcher: A directory containing the AWS Lambda function. - * ecs-worker-launcher.js: A Lambda function that sends event data into Amazon SQS and starts an Amazon ECS Task. -* fabfile.py: A Python Fabric script that configures all of the necessary components for this demo. -* config.py: User-specific constants for fabfile.py. Edit these with your own values. -* requirements.py: Python requirements file for fabfile.py. -* LambdaECSWorkerPattern.png: The image you see above. - -## Prerequisites - -* Pick an AWS Region that support AWS Lambda, Amazon ECS and Amazon SQS. Check the [AWS Region - Table](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/) to see a list of AWS - services supported by AWS Region. -* You need the AWS CLI installed and configured. The [Setting Up section of the Amazon ECS - documentation](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/get-set-up-for-amazon-ecs.html) includes - instructions on how to set up the AWS CLI. -* The AWS IAM user configured to use with the AWS CLI needs permissions that allow creating and configuring Amazon S3 - buckets, Amazon SQS queues, AWS Lambda functions, IAM Roles and policies, and Amazon ECS task definitions. -* You should be familiar with running AWS Lambda functions. Check out the [Getting Started 2: Handling Amazon S3 Events - Using the AWS Lambda Console - (Node.js)](http://docs.aws.amazon.com/lambda/latest/dg/getting-started-amazons3-events.html) guide to familiarize - yourself with AWS Lambda and how to trigger Lambda functions from Amazon S3 event notifications. -* You should also understand the basic concepts behind Amazon EC2 Container Service (ECS). The [Getting Started with - Amazon ECS](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_GetStarted.html) guide is a good place to - start. Use this guide to start a default Amazon ECS cluster, if you haven’t already. -* Some familiarity with Docker files, creating Docker images and using DockerHub would also be helpful. See the [Docker - User Guide](https://docs.docker.com/userguide/) to get you acquainted with this. -* This demo uses DockerHub to manage and store Docker images. You can [set up a DockerHub account for - free](https://hub.docker.com/) or modify the fabfile.py script to use a Docker repository of your own. - Keep your DockerHub or your own Docker repository’s credentials handy. -* We assume that the ECS cluster you’re using is this demo on is for testing and does not run critical applications. - In particular, we’ll use one of the cluster nodes to build a docker image which may temporarily consume CPU and - memory resources in addition to the containers you’re running on the cluster. This is for purposes of simplifying - the demo and should not be done on production systems. Make sure you have enough spare RAM and CPU resources to - prepare a new Docker image from scratch on its node(s). A simple "default" cluster with a single t2.micro instance - set up as described in the ECS documentation is sufficient. - -## How to setup - -The fabfile.py Python script included in this repository comes with all commands necessary to set up and run this -sample application. You can examine the script to identify individual steps, then run -fab individually, or you can simply run fab setup to get everything set -up for you. - -Here’s how to set up: - - # 1. Clone this repository into a local directory. - - git clone https://github.com/awslabs/lambda-ecs-worker-pattern.git - - # 2. Edit config.py and use your own values there for your name and email address, DockerHub account details, - # AWS region, SSH key name and other details. - - # 3. Make sure you have Python2 installed. Depending on your installation, - # you may need to use the pip2 command below, or you can use the plain pip command if it points to the Python2 - # version of PIP. - - sudo pip2 install -r requirements.txt - - # 4. Now, configure the AWS CLI if you haven’t already. This script will use your AWS credentials and other - # information from your AWS CLI configuration profile(s). - - fab setup - - # At some stage, the script will ask you for your DockerHub login name and password, in order to upload a docker - # image to DockerHub and configure your ECS cluster to access your DockerHub private repository. - - # 5. Take a note of the bucket name mentioned at the end of the setup. - - # 6. Go to the AWS console, upload ECSLogo.zip to this bucket and wait a minute. - - # 7. Check the Lambda logs for any errors. If everything goes well, you should see a .PNG image in the same - # bucket a few minutes after uploading. - -Enjoy! - diff --git a/config.py b/config.py deleted file mode 100644 index 9993729..0000000 --- a/config.py +++ /dev/null @@ -1,15 +0,0 @@ -# Constants (User configurable) - -FULL_NAME_AND_EMAIL = 'First Last ' # For Dockerfile/POV-Ray builds. -APP_NAME = 'ECSPOVRayWorker' # Used to generate derivative names unique to the application. - -DOCKERHUB_USER = 'username' -DOCKERHUB_EMAIL = 'email@domain.com' -DOCKERHUB_REPO = 'private' -DOCKERHUB_TAG = DOCKERHUB_USER + '/' + DOCKERHUB_REPO + ':' + APP_NAME - -AWS_REGION = 'us-east-1' -AWS_PROFILE = 'default' # The same profile used by your AWS CLI installation - -SSH_KEY_NAME = 'your-ssh-key.pem' # Expected to be in ~/.ssh -ECS_CLUSTER = 'default' diff --git a/ecs-worker-launcher/config.sample.json b/ecs-worker-launcher/config.sample.json deleted file mode 100644 index 4e4d526..0000000 --- a/ecs-worker-launcher/config.sample.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "queue": "https://.queue.amazonaws.com//ECSPOVRayWorkerQueue", - "task": "", - "s3_key_suffix_whitelist": [".zip"] -} diff --git a/ecs-worker-launcher/ecs-worker-launcher.js b/ecs-worker-launcher/ecs-worker-launcher.js deleted file mode 100644 index 8280a48..0000000 --- a/ecs-worker-launcher/ecs-worker-launcher.js +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). -// You may not use this file except in compliance with the License. -// A copy of the License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. -// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and limitations under the License. - -// This AWS Lambda function forwards the given event data into an Amazon SQS queue, then starts an Amazon ECS task to -// process that event. - -var fs = require('fs'); -var async = require('async'); -var aws = require('aws-sdk'); -var sqs = new aws.SQS({apiVersion: '2012-11-05'}); -var ecs = new aws.ECS({apiVersion: '2014-11-13'}); - -// Check if the given key suffix matches a suffix in the whitelist. Return true if it matches, false otherwise. -exports.checkS3SuffixWhitelist = function(key, whitelist) { - if(!whitelist){ return true; } - if(typeof whitelist == 'string'){ return key.match(whitelist + '$') } - if(Object.prototype.toString.call(whitelist) === '[object Array]') { - for(var i = 0; i < whitelist.length; i++) { - if(key.match(whitelist[i] + '$')) { return true; } - } - return false; - } - console.log( - 'Unsupported whitelist type (' + Object.prototype.toString.call(whitelist) + - ') for: ' + JSON.stringify(whitelist) - ); - return false; -}; - -exports.handler = function(event, context) { - console.log('Received event:'); - console.log(JSON.stringify(event, null, ' ')); - - var config = JSON.parse(fs.readFileSync('config.json', 'utf8')); - if(!config.hasOwnProperty('s3_key_suffix_whitelist')) { - config.s3_key_suffix_whitelist = false; - } - console.log('Config: ' + JSON.stringify(config)); - - var key = event.Records[0].s3.object.key; - - if(!exports.checkS3SuffixWhitelist(key, config.s3_key_suffix_whitelist)) { - context.fail('Suffix for key: ' + key + ' is not in the whitelist') - } - - // We can now go on. Put the Amazon S3 URL into Amazon SQS and start an Amazon ECS task to process it. - async.waterfall([ - function (next) { - var params = { - MessageBody: JSON.stringify(event), - QueueUrl: config.queue - }; - sqs.sendMessage(params, function (err, data) { - if (err) { console.warn('Error while sending message: ' + err); } - else { console.info('Message sent, ID: ' + data.MessageId); } - next(err); - }); - }, - function (next) { - // Starts an ECS task to work through the feeds. - var params = { - taskDefinition: config.task, - count: 1, - cluster: 'default' - }; - ecs.runTask(params, function (err, data) { - if (err) { console.warn('error: ', "Error while starting task: " + err); } - else { console.info('Task ' + config.task + ' started: ' + JSON.stringify(data.tasks))} - next(err); - }); - } - ], function (err) { - if (err) { - context.fail('An error has occurred: ' + err); - } - else { - context.succeed('Successfully processed Amazon S3 URL.'); - } - } - ); -}; diff --git a/ecs-worker/ecs-worker.sh b/ecs-worker/ecs-worker.sh deleted file mode 100644 index 55d9d97..0000000 --- a/ecs-worker/ecs-worker.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. -# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and limitations under the License. - -# -# Simple POV-Ray worker shell script. -# -# Uses the AWS CLI utility to fetch a message from SQS, fetch a ZIP file from S3 that was specified in the message, -# render its contents with POV-Ray, then upload the resulting .png file to the same S3 bucket. -# - -region=${AWS_REGION} -queue=${SQS_QUEUE_URL} - -# Fetch messages and render them until the queue is drained. -while [ /bin/true ]; do - # Fetch the next message and extract the S3 URL to fetch the POV-Ray source ZIP from. - echo "Fetching messages fom SQS queue: ${queue}..." - result=$( \ - aws sqs receive-message \ - --queue-url ${queue} \ - --region ${region} \ - --wait-time-seconds 20 \ - --query Messages[0].[Body,ReceiptHandle] \ - | sed -e 's/^"\(.*\)"$/\1/'\ - ) - - if [ -z "${result}" ]; then - echo "No messages left in queue. Exiting." - exit 0 - else - echo "Message: ${result}." - - receipt_handle=$(echo ${result} | sed -e 's/^.*"\([^"]*\)"\s*\]$/\1/') - echo "Receipt handle: ${receipt_handle}." - - bucket=$(echo ${result} | sed -e 's/^.*arn:aws:s3:::\([^\\]*\)\\".*$/\1/') - echo "Bucket: ${bucket}." - - key=$(echo ${result} | sed -e 's/^.*\\"key\\":\s*\\"\([^\\]*\)\\".*$/\1/') - echo "Key: ${key}." - - base=${key%.*} - ext=${key##*.} - - if [ \ - -n "${result}" -a \ - -n "${receipt_handle}" -a \ - -n "${key}" -a \ - -n "${base}" -a \ - -n "${ext}" -a \ - "${ext}" = "zip" \ - ]; then - mkdir -p work - pushd work - - echo "Copying ${key} from S3 bucket ${bucket}..." - aws s3 cp s3://${bucket}/${key} . --region ${region} - - echo "Unzipping ${key}..." - unzip ${key} - - if [ -f ${base}.ini ]; then - echo "Rendering POV-Ray scene ${base}..." - if povray ${base}; then - if [ -f ${base}.png ]; then - echo "Copying result image ${base}.png to s3://${bucket}/${base}.png..." - aws s3 cp ${base}.png s3://${bucket}/${base}.png - else - echo "ERROR: POV-Ray source did not generate ${base}.png image." - fi - else - echo "ERROR: POV-Ray source did not render successfully." - fi - else - echo "ERROR: No ${base}.ini file found in POV-Ray source archive." - fi - - echo "Cleaning up..." - popd - /bin/rm -rf work - - echo "Deleting message..." - aws sqs delete-message \ - --queue-url ${queue} \ - --region ${region} \ - --receipt-handle "${receipt_handle}" - - else - echo "ERROR: Could not extract S3 bucket and key from SQS message." - fi - fi -done diff --git a/fabfile.py b/fabfile.py deleted file mode 100644 index 47d8567..0000000 --- a/fabfile.py +++ /dev/null @@ -1,792 +0,0 @@ -#!/usr/bin/python -# coding: utf-8 - -# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. -# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and limitations under the License. - -# -# fabfile.py -# - -# -# A Python fabric file for setting up and managing the AWS ECS POV-Ray worker demo. -# This file assumes python2 is installed on the system as the default. -# - -# Imports -from fabric.api import local, quiet, env, run, put, cd -from ConfigParser import ConfigParser -import boto -import boto.s3 -from boto.exception import BotoServerError -from zipfile import ZipFile, ZIP_DEFLATED -import os -import json -import time -from urllib2 import unquote -from cStringIO import StringIO - -# Constants (User configurable), imported from config.py - -from config import * - -# Constants (Application specific) -BUCKET_POSTFIX = '-pov-ray-bucket' # Gets put after the unix user ID to create the bucket name. -SSH_KEY_DIR = os.environ['HOME'] + '/.ssh' -SQS_QUEUE_NAME = APP_NAME + 'Queue' -LAMBDA_FUNCTION_NAME = 'ecs-worker-launcher' -LAMBDA_FUNCTION_DEPENDENCIES = 'async' -ECS_TASK_NAME = APP_NAME + 'Task' - -# Constants (OS specific) -USER = os.environ['HOME'].split('/')[-1] -AWS_BUCKET = USER + BUCKET_POSTFIX -AWS_CONFIG_FILE_NAME = os.environ['HOME'] + '/.aws/config' -AWS_CREDENTIAL_FILE_NAME = os.environ['HOME'] + '/.aws/credentials' - -# Constants -AWS_CLI_STANDARD_OPTIONS = ( - ' --region ' + AWS_REGION + - ' --profile ' + AWS_PROFILE + - ' --output json' -) - -SSH_USER = 'ec2-user' -CPU_SHARES = 512 # POV-Ray needs at least half a CPU to work nicely. -MEMORY = 512 -ZIPFILE_NAME = LAMBDA_FUNCTION_NAME + '.zip' - -BUCKET_PERMISSION_SID = APP_NAME + 'Permission' -WAIT_TIME = 5 # seconds to allow for eventual consistency to kick in. -RETRIES = 5 # Number of retries before we give up on something. - -# Templates and embedded scripts - -LAMBDA_EXECUTION_ROLE_NAME = APP_NAME + '-Lambda-Execution-Role' -LAMBDA_EXECUTION_ROLE_POLICY_NAME = 'AWSLambdaExecutionPolicy' -LAMBDA_EXECUTION_ROLE_POLICY = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "logs:*", - "sqs:SendMessage", - "ecs:RunTask" - ], - "Resource": [ - "arn:aws:logs:*:*:*", - "arn:aws:sqs:*:*:*", - "arn:aws:ecs:*:*:*" - ] - } - ] -} - -LAMBDA_EXECUTION_ROLE_TRUST_POLICY = { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} - -LAMBDA_FUNCTION_CONFIG = { - "s3_key_suffix_whitelist": ['.zip'], # Only S3 keys with this URL will be accepted. - "queue": '', # To be filled in with the queue ARN. - "task": ECS_TASK_NAME -} - -LAMBDA_FUNCTION_CONFIG_PATH = './' + LAMBDA_FUNCTION_NAME + '/config.json' - -BUCKET_NOTIFICATION_CONFIGURATION = { - "LambdaFunctionConfigurations": [ - { - "Id": APP_NAME, - "LambdaFunctionArn": "", - "Events": [ - "s3:ObjectCreated:*" - ] - } - ] -} - -ECS_ROLE_BUCKET_ACCESS_POLICY_NAME = APP_NAME + "BucketAccessPolicy" -ECS_ROLE_BUCKET_ACCESS_POLICY = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListAllMyBuckets" - ], - "Resource": "arn:aws:s3:::*" - }, - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:GetBucketLocation" - ], - "Resource": "" # To be filled in by a function below. - }, - { - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject" - ], - "Resource": "" # To be filled in by a function below. - } - ] -} - -WORKER_PATH = 'ecs-worker' -WORKER_FILE = 'ecs-worker.sh' - -DOCKERFILE = """ -# POV-Ray Amazon ECS Worker - -FROM ubuntu:14.04 - -MAINTAINER %(name)s - -# Libraries and dependencies - -RUN \ - apt-get update && apt-get -y install \ - autoconf \ - build-essential \ - git \ - libboost-thread-dev \ - libjpeg-dev \ - libopenexr-dev \ - libpng-dev \ - libtiff-dev \ - python \ - python-dev \ - python-distribute \ - python-pip \ - unzip \ - zlib1g-dev - -# Compile and install POV-Ray - -RUN \ - mkdir /src && \ - cd /src && \ - git clone https://github.com/POV-Ray/povray.git && \ - cd povray && \ - git checkout origin/3.7-stable && \ - cd unix && \ - sed 's/automake --w/automake --add-missing --w/g' -i prebuild.sh && \ - sed 's/dist-bzip2/dist-bzip2 subdir-objects/g' -i configure.ac && \ - ./prebuild.sh && \ - cd .. && \ - ./configure COMPILED_BY="%(name)s" LIBS="-lboost_system -lboost_thread" && \ - make && \ - make install - -# Install AWS CLI - -RUN \ - pip install awscli - -WORKDIR / - -COPY %(worker_file)s / - -CMD [ "./%(worker_file)s" ] -""" - -TASK_DEFINITION = { - "family": APP_NAME, - "containerDefinitions": [ - { - "environment": [ - { - "name": "AWS_REGION", - "value": AWS_REGION - } - ], - "name": APP_NAME, - "image": DOCKERHUB_TAG, - "cpu": CPU_SHARES, - "memory": MEMORY, - "essential": True - } - ] -} - -POV_RAY_SCENE_NAME = 'ECSLogo' -POV_RAY_SCENE_FILE = POV_RAY_SCENE_NAME + '.zip' -POV_RAY_SCENE_FILES = [ - 'AWS_Logo_PoweredBy_300px.png', - 'ECSLogo.ini', - 'ECSLogo.pov' -] - -# Functions - - -# Dependencies and credentials. - - -def update_dependencies(): - local('pip2 install -r requirements.txt') - local('cd ' + LAMBDA_FUNCTION_NAME + '; npm install ' + LAMBDA_FUNCTION_DEPENDENCIES) - - -def get_aws_credentials(): - config = ConfigParser() - config.read(AWS_CONFIG_FILE_NAME) - config.read(AWS_CREDENTIAL_FILE_NAME) - return config.get(AWS_PROFILE, 'aws_access_key_id'), config.get(AWS_PROFILE, 'aws_secret_access_key') - -# AWS IAM - - -def get_iam_connection(): - aws_access_key_id, aws_secret_access_key = get_aws_credentials() - return boto.connect_iam(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - -# AWS Lambda - - -def dump_lambda_function_configuration(): - print('Writing config for Lambda function...') - lambda_function_config = LAMBDA_FUNCTION_CONFIG.copy() - lambda_function_config['queue'] = get_queue_url() - with open(LAMBDA_FUNCTION_CONFIG_PATH, 'w') as fp: - fp.write(json.dumps(lambda_function_config)) - - -def create_lambda_deployment_package(): - print('Creating ZIP file: ' + ZIPFILE_NAME + '...') - with ZipFile(ZIPFILE_NAME, 'w', ZIP_DEFLATED) as z: - saved_dir = os.getcwd() - os.chdir(LAMBDA_FUNCTION_NAME) - for root, dirs, files in os.walk('.'): - for basename in files: - filename = os.path.join(root, basename) - print('Adding: ' + filename + '...') - z.write(filename) - os.chdir(saved_dir) - z.close() - - -def get_or_create_lambda_execution_role(): - iam = get_iam_connection() - target_policy = json.dumps(LAMBDA_EXECUTION_ROLE_TRUST_POLICY, sort_keys=True) - - try: - result = iam.get_role(LAMBDA_EXECUTION_ROLE_NAME) - print('Found role: ' + LAMBDA_EXECUTION_ROLE_NAME + '.') - - policy = result['get_role_response']['get_role_result']['role']['assume_role_policy_document'] - if ( - policy is not None and - json.dumps(json.loads(unquote(policy)), sort_keys=True) == target_policy - ): - print('Assume role policy for: ' + LAMBDA_EXECUTION_ROLE_NAME + ' verified.') - else: - print('Updating assume role policy for: ' + LAMBDA_EXECUTION_ROLE_NAME + '.') - iam.update_assume_role_policy(LAMBDA_EXECUTION_ROLE_NAME, target_policy) - time.sleep(WAIT_TIME) - except BotoServerError: - print('Creating role: ' + LAMBDA_EXECUTION_ROLE_NAME + '...') - iam.create_role( - LAMBDA_EXECUTION_ROLE_NAME, - assume_role_policy_document=target_policy - ) - result = iam.get_role(LAMBDA_EXECUTION_ROLE_NAME) - - role_arn = result['get_role_response']['get_role_result']['role']['arn'] - - return role_arn - - -def check_lambda_execution_role_policies(): - iam = get_iam_connection() - - response = iam.list_role_policies(LAMBDA_EXECUTION_ROLE_NAME) - policy_names = response['list_role_policies_response']['list_role_policies_result']['policy_names'] - - found = False - for p in policy_names: - found = (p == LAMBDA_EXECUTION_ROLE_POLICY_NAME) - if found: - print('Found policy: ' + LAMBDA_EXECUTION_ROLE_POLICY_NAME + '.') - break - - if not found: - print('Attaching policy: ' + LAMBDA_EXECUTION_ROLE_POLICY_NAME + '.') - iam.put_role_policy( - LAMBDA_EXECUTION_ROLE_NAME, - 'AWSLambdaExecute', - json.dumps(LAMBDA_EXECUTION_ROLE_POLICY) - ) - - return - - -def get_lambda_function_arn(): - result = json.loads( - local( - 'aws lambda list-functions' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - ) - if result is not None and isinstance(result, dict): - for f in result.get('Functions', []): - if f['FunctionName'] == LAMBDA_FUNCTION_NAME: - return f['FunctionArn'] - - return None - - -def delete_lambda_function(): - local( - 'aws lambda delete-function' + - ' --function-name ' + LAMBDA_FUNCTION_NAME + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - -def update_lambda_function(): - dump_lambda_function_configuration() - create_lambda_deployment_package() - role_arn = get_or_create_lambda_execution_role() - check_lambda_execution_role_policies() - - if get_lambda_function_arn() is not None: - print('Deleting existing Lambda function ' + LAMBDA_FUNCTION_NAME + '.') - delete_lambda_function() - - local( - 'aws lambda create-function' + - ' --function-name ' + LAMBDA_FUNCTION_NAME + - ' --zip-file fileb://./' + ZIPFILE_NAME + - ' --role ' + role_arn + - ' --handler ' + LAMBDA_FUNCTION_NAME + '.handler' + - ' --runtime nodejs' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - -def show_lambda_execution_role_policy(): - print json.dumps(LAMBDA_EXECUTION_ROLE_POLICY, sort_keys=True) - - -# Amazon S3 - - -def get_s3_connection(): - aws_access_key_id, aws_secret_access_key = get_aws_credentials() - return boto.s3.connect_to_region( - AWS_REGION, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key - ) - - -def get_or_create_bucket(): - s3 = get_s3_connection() - b = s3.lookup(AWS_BUCKET) - if b is None: - print('Creating bucket: ' + AWS_BUCKET + ' in region: ' + AWS_REGION + '...') - LOCATION = AWS_REGION if AWS_REGION != 'us-east-1' else '' - b = s3.create_bucket(AWS_BUCKET, location=LOCATION) - else: - print('Found bucket: ' + AWS_BUCKET + '.') - - return b - - -def check_bucket_permissions(): - with quiet(): - result = local( - 'aws lambda get-policy' + - ' --function-name ' + LAMBDA_FUNCTION_NAME + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - if result.failed or result == '': - return False - - result_decoded = json.loads(result) - if not isinstance(result_decoded, dict): - return False - - policy = json.loads(result_decoded.get('Policy', '{}')) - if not isinstance(policy, dict): - return False - - statements = policy.get('Statement', []) - for s in statements: - if s.get('Sid', '') == BUCKET_PERMISSION_SID: - return True - - return False - - -def update_bucket_permissions(): - get_or_create_bucket() - if check_bucket_permissions(): - print('Lambda invocation permission for bucket: ' + AWS_BUCKET + ' is set.') - else: - print('Setting Lambda invocation permission for bucket: ' + AWS_BUCKET + '.') - local( - 'aws lambda add-permission' + - ' --function-name ' + LAMBDA_FUNCTION_NAME + - ' --region ' + AWS_REGION + - ' --statement-id ' + BUCKET_PERMISSION_SID + - ' --action "lambda:InvokeFunction"' + - ' --principal s3.amazonaws.com' + - ' --source-arn arn:aws:s3:::' + AWS_BUCKET + - ' --profile ' + AWS_PROFILE, - capture=True - ) - - -def check_bucket_notifications(): - result = local( - 'aws s3api get-bucket-notification-configuration' + - ' --bucket ' + AWS_BUCKET + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - if result.failed or result == '': - return False - - result_decoded = json.loads(result) - if not isinstance(result_decoded, dict): - return False - - -def setup_bucket_notifications(): - update_lambda_function() - update_bucket_permissions() - - notification_configuration = BUCKET_NOTIFICATION_CONFIGURATION.copy() - lambda_function_arn = get_lambda_function_arn() - notification_configuration['LambdaFunctionConfigurations'][0]['LambdaFunctionArn'] = lambda_function_arn - - if check_bucket_notifications(): - print('Bucket notification configuration for bucket: ' + AWS_BUCKET + ' is set.') - else: - print('Setting bucket notification configuration for bucket: ' + AWS_BUCKET + '.') - local( - 'aws s3api put-bucket-notification-configuration' + - ' --bucket ' + AWS_BUCKET + - ' --notification-configuration \'' + json.dumps(notification_configuration, sort_keys=True) + '\'' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - -def show_bucket_name(): - print("Your bucket name is: " + AWS_BUCKET) - - -# Amazon EC2 - -def get_instance_ip_from_id(instance_id): - result = json.loads(local( - 'aws ec2 describe-instances' + - ' --instance ' + instance_id + - ' --query Reservations[0].Instances[0].PublicIpAddress' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - )) - print ('IP address for instance ' + instance_id + ' is: ' + result) - return result - - -def get_instance_profile_name(instance_id): - result = json.loads(local( - 'aws ec2 describe-instances' + - ' --instance ' + instance_id + - ' --query Reservations[0].Instances[0].IamInstanceProfile.Arn' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - )).split('/')[-1] - print('IAM instance profile for instance ' + instance_id + ' is: ' + result) - return result - - -def get_instance_role(instance_id): - profile = get_instance_profile_name(instance_id) - result = json.loads(local( - 'aws iam get-instance-profile' + - ' --instance-profile-name ' + profile + - ' --query InstanceProfile.Roles[0].RoleName' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - )) - print('Role for instance ' + instance_id + ' is: ' + result) - return result - - -# Amazon ECS - - -def get_container_instances(): - result = json.loads(local( - 'aws ecs list-container-instances' + - ' --query containerInstanceArns' + - ' --cluster ' + ECS_CLUSTER + - AWS_CLI_STANDARD_OPTIONS, - capture=True - )) - print('Container instances: ' + ','.join(result)) - return result - - -def get_first_ecs_instance(): - container_instances = get_container_instances() - - result = json.loads(local( - 'aws ecs describe-container-instances' + - ' --cluster ' + ECS_CLUSTER + - ' --container-instances ' + container_instances[0] + - ' --query containerInstances[0].ec2InstanceId' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - )) - print('First container instance: ' + result) - return result - - -def get_first_ecs_instance_ip(): - i = get_first_ecs_instance() - return get_instance_ip_from_id(i) - - -def prepare_env(): - env.host_string = get_first_ecs_instance_ip() - env.user = SSH_USER - env.key_filename = SSH_KEY_DIR + '/' + SSH_KEY_NAME - - -def generate_dockerfile(): - return DOCKERFILE % {'name': FULL_NAME_AND_EMAIL, 'worker_file': WORKER_FILE} - - -def show_dockerfile(): - print generate_dockerfile() - - -def update_ecs_image(): - prepare_env() - run('mkdir -p ' + APP_NAME) - - dockerfile_string = generate_dockerfile() - dockerfile = StringIO(dockerfile_string) - put(dockerfile, remote_path='~/' + APP_NAME + '/Dockerfile', mode=0644) - - with open(os.path.join(WORKER_PATH, WORKER_FILE), "r") as fp: - put(fp, remote_path='~/' + APP_NAME + '/' + WORKER_FILE, mode=0755) - - # Build the docker image and upload it to Dockerhub. This will prompt the user for their password. - with cd('~/' + APP_NAME): - run('docker build -t ' + DOCKERHUB_TAG + ' .') - #run('docker login -u ' + DOCKERHUB_USER + ' -e ' + DOCKERHUB_EMAIL) - login_str = local('aws ecr get-login', capture=True) - print(login_str) - run('%s' % login_str) - run('docker push ' + DOCKERHUB_TAG) - - # Cleanup. - run('/bin/rm -rf ' + APP_NAME) - - -def generate_task_definition(): - task_definition = TASK_DEFINITION.copy() - task_definition['containerDefinitions'][0]['environment'].append( - { - 'name': 'SQS_QUEUE_URL', - 'value': get_queue_url() - } - ) - return task_definition - - -def show_task_definition(): - print json.dumps(generate_task_definition(), indent=4) - - -def update_ecs_task_definition(): - task_definition_string = json.dumps(generate_task_definition()) - - local( - 'aws ecs register-task-definition' + - ' --family ' + ECS_TASK_NAME + - ' --cli-input-json \'' + task_definition_string + '\'' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - -def generate_ecs_role_policy(): - result = ECS_ROLE_BUCKET_ACCESS_POLICY.copy() - result['Statement'][1]['Resource'] = 'arn:aws:s3:::' + AWS_BUCKET - result['Statement'][2]['Resource'] = 'arn:aws:s3:::' + AWS_BUCKET + '/*' - return result - - -def show_ecs_role_policy(): - policy = generate_ecs_role_policy() - print json.dumps(policy, indent=4, sort_keys=True) - - -def check_ecs_role_policy(): - role = get_instance_role(get_first_ecs_instance()) - iam = get_iam_connection() - - policy = None - # noinspection PyBroadException - try: - response = iam.get_role_policy(role, ECS_ROLE_BUCKET_ACCESS_POLICY_NAME) - policy_raw = response['get_role_policy_response']['get_role_policy_result']['policy_document'] - policy = json.dumps(json.loads(unquote(policy_raw)), sort_keys=True) - except Exception: - pass - - if policy is None: - print('ECS role policy ' + role + 'is missing.') - return False - else: - target_policy = json.dumps(generate_ecs_role_policy(), sort_keys=True) - if policy == target_policy: - print('ECS role policy ' + role + ' is present and correct.') - return True - else: - print('ECS role policy ' + role + ' is present, but different.') - return False - - -def update_ecs_role_policy(): - if check_ecs_role_policy(): - return True - else: - role = get_instance_role(get_first_ecs_instance()) - policy = json.dumps(generate_ecs_role_policy()) - iam = get_iam_connection() - print('Putting policy: ' + ECS_ROLE_BUCKET_ACCESS_POLICY_NAME + ' into role: ' + role) - iam.put_role_policy( - role, - ECS_ROLE_BUCKET_ACCESS_POLICY_NAME, - policy - ) - - -# Amazon SQS - - -def get_queue_url(): - result = local( - 'aws sqs list-queues' + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - if result is not None and result != '': - result_struct = json.loads(result) - if isinstance(result_struct, dict) and 'QueueUrls' in result_struct: - for u in result_struct['QueueUrls']: - if u.split('/')[-1] == SQS_QUEUE_NAME: - return u - - return None - - -def get_or_create_queue(): - u = get_queue_url() - if u is None: - local( - 'aws sqs create-queue' + - ' --queue-name ' + SQS_QUEUE_NAME + - AWS_CLI_STANDARD_OPTIONS, - capture=True - ) - - tries = 0 - while True: - time.sleep(WAIT_TIME) - u = get_queue_url() - - if u is not None and tries < RETRIES: - return u - - tries += 1 - - -# Putting together the demo POV-Ray file. - -def create_pov_ray_zip(): - if os.path.exists(POV_RAY_SCENE_FILE): - print('Deleting old ZIP file: ' + POV_RAY_SCENE_FILE) - os.remove(POV_RAY_SCENE_FILE) - - print('Creating ZIP file: ' + POV_RAY_SCENE_FILE + '...') - with ZipFile(POV_RAY_SCENE_FILE, 'w', ZIP_DEFLATED) as z: - saved_dir = os.getcwd() - os.chdir(POV_RAY_SCENE_NAME) - for f in POV_RAY_SCENE_FILES: - print('Adding: ' + f + '...') - z.write(f) - os.chdir(saved_dir) - z.close() - -# High level functions. Call these as "fab " - - -def update_bucket(): - get_or_create_bucket() - - -def update_lambda(): - update_lambda_function() - update_bucket_permissions() - setup_bucket_notifications() - - -def update_ecs(): - update_ecs_image() - update_ecs_task_definition() - - -def update_queue(): - get_or_create_queue() - - -def setup(): - update_dependencies() - update_bucket() - update_queue() - update_lambda() - update_ecs() - update_ecs_role_policy() - create_pov_ray_zip() - show_bucket_name() diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 99a6966..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -fabric>=1.10.0 -boto>=2.38.0 \ No newline at end of file