diff --git a/README.md b/README.md index f002d82..8dd8941 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,25 @@ Example stack name: RenesasPipeline Expected build time: 27min / rebuild (without any change, just use sstate cache): 9min ---- +### A AWS CodeBuild Project +This will create an Embedded Linux ready AWS CodeBuild project that can be used to connect to a source, e.g. [GitHub Actions](https://docs.aws.amazon.com/codebuild/latest/userguide/action-runner.html). This is not using any CodePipeline. + +And use the EFS to share downloads and sstate cache between the runners. + +The connection to the CodeBuild source must be performed manually. + +Also you can clone the CodeBuild project and share the efs between the CodeBuild projects. + +See the AWS CodeBuild pipeline: EmbeddedLinuxCodebuildProje-* + +To make a source connection to GitHub you need to: +- Select a "Source provider"->"GitHub" +- Select "Primary source webhook events" -> "Webhook - optional" -> "Rebuild every time a code change is pushed to this repository" +- Add "Filter group 1" -> "WORKFLOW_JOB_QUEUED" +- Modify the GitHub action `runs-on: ${{ vars.CODEBUILD_RUNNER_NAME }}-${{ github.run_id }}-${{ github.run_attempt }}` +CODEBUILD_RUNNER_NAME should be `codebuild-EmbeddedLinuxCodebuildProjeNAME` with prefix `codebuild-`. See example [here](https://github.com/aws4embeddedlinux/meta-aws-demos/blob/master/.github/workflows/build-gg.yml). + +Example stack name: EmbeddedLinuxCodeBuildProject ## Useful NPM and CDK commands diff --git a/bin/app.ts b/bin/app.ts index 3e166f1..420fe00 100644 --- a/bin/app.ts +++ b/bin/app.ts @@ -3,6 +3,7 @@ import * as cdk from "aws-cdk-lib"; import { addDependency } from "aws-cdk-lib/core/lib/deps"; import { EmbeddedLinuxPipelineStack, + EmbeddedLinuxCodebuildProjectStack, BuildImageDataStack, BuildImagePipelineStack, BuildImageRepoStack, @@ -10,6 +11,9 @@ import { ImageKind, ProjectKind, } from "aws4embeddedlinux-cdk-lib"; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { RemovalPolicy } from 'aws-cdk-lib'; +import * as kms from 'aws-cdk-lib/aws-kms'; const app = new cdk.App(); @@ -29,6 +33,47 @@ const defaultProps: cdk.StackProps = { env, }; +/** + * Set up networking to allow us to securely attach EFS to our CodeBuild instances. + */ +const vpc = new PipelineNetworkStack(app, "PipelineNetwork", { + ...defaultProps, +}); + +/** + * Set up shared Artifacts and ArtifactAccessLogging Bucket for all example pipelines. + * Using Pipeline Network Stack as a container for the buckets. + */ + +const accessLoggingBucket = new s3.Bucket(vpc, 'ArtifactAccessLogging', { + versioned: true, + enforceSSL: true, +}); + +const encryptionKey = new kms.Key(vpc, 'PipelineArtifactKey', { + removalPolicy: RemovalPolicy.DESTROY, + enableKeyRotation: true, +}); + +const artifactBucket = new s3.Bucket(vpc, 'PipelineArtifacts', { + versioned: true, + enforceSSL: true, + serverAccessLogsBucket: accessLoggingBucket, + serverAccessLogsPrefix: "PipelineArtifacts", + encryptionKey, + encryption: s3.BucketEncryption.KMS, + blockPublicAccess: new s3.BlockPublicAccess( + s3.BlockPublicAccess.BLOCK_ALL + ), +}); + +const outputBucket = new s3.Bucket(vpc, 'PipelineOutput', { + versioned: true, + enforceSSL: true, + serverAccessLogsBucket: accessLoggingBucket, + serverAccessLogsPrefix: "PipelineOutput", +}); + /** * Set up the Stacks that create our Build Host. */ @@ -46,13 +91,9 @@ const buildImagePipeline = new BuildImagePipelineStack(app, "BuildImagePipeline" dataBucket: buildImageData.bucket, repository: buildImageRepo.repository, imageKind: ImageKind.Ubuntu22_04, -}); - -/** - * Set up networking to allow us to securely attach EFS to our CodeBuild instances. - */ -const vpc = new PipelineNetworkStack(app, { - ...defaultProps, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "BuildImagePipeline", + artifactBucket: artifactBucket, }); /** @@ -63,6 +104,11 @@ const pokyPipeline = new EmbeddedLinuxPipelineStack(app, "PokyPipeline", { imageRepo: buildImageRepo.repository, imageTag: ImageKind.Ubuntu22_04, vpc: vpc.vpc, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "PokyPipeline", + artifactBucket: artifactBucket, + outputBucket: outputBucket, + subDirectoryName: "PokyPipeline", }); pokyPipeline.addDependency(buildImagePipeline) @@ -76,6 +122,11 @@ const qemuEmbeddedLinuxPipeline = new EmbeddedLinuxPipelineStack(app, "QemuEmbed vpc: vpc.vpc, layerRepoName: "qemu-demo-layer-repo", projectKind: ProjectKind.MetaAwsDemo, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "QemuEmbeddedLinuxPipeline", + artifactBucket: artifactBucket, + outputBucket: outputBucket, + subDirectoryName: "QemuEmbeddedLinuxPipeline", }); qemuEmbeddedLinuxPipeline.addDependency(buildImagePipeline) @@ -89,6 +140,10 @@ const pokyAmiPipeline = new EmbeddedLinuxPipelineStack(app, "PokyAmiPipeline", { vpc: vpc.vpc, layerRepoName: "ec2-ami-poky-layer-repo", projectKind: ProjectKind.PokyAmi, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "PokyAmiPipeline", + artifactBucket: artifactBucket, + subDirectoryName: "PokyAmiPipeline", }); pokyAmiPipeline.addDependency(buildImagePipeline) @@ -102,6 +157,11 @@ const kasPipeline = new EmbeddedLinuxPipelineStack(app, "KasPipeline", { vpc: vpc.vpc, layerRepoName: "biga-kas-layer-repo", projectKind: ProjectKind.Kas, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "KasPipeline", + artifactBucket: artifactBucket, + outputBucket: outputBucket, + subDirectoryName: "KasPipeline", }); kasPipeline.addDependency(buildImagePipeline) @@ -115,6 +175,11 @@ const renesasPipeline = new EmbeddedLinuxPipelineStack(app, "RenesasPipeline", { vpc: vpc.vpc, layerRepoName: "renesas-layer-repo", projectKind: ProjectKind.Renesas, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "RenesasPipeline", + artifactBucket: artifactBucket, + outputBucket: outputBucket, + subDirectoryName: "RenesasPipeline", }); renesasPipeline.addDependency(buildImagePipeline) @@ -128,5 +193,22 @@ const nxpImxPipeline = new EmbeddedLinuxPipelineStack(app, "NxpImxPipeline", { vpc: vpc.vpc, layerRepoName: "nxp-imx-layer-repo", projectKind: ProjectKind.NxpImx, + accessLoggingBucket: accessLoggingBucket, + serverAccessLogsPrefix: "NxpImxPipeline", + artifactBucket: artifactBucket, + outputBucket: outputBucket, + subDirectoryName: "NxpImxPipeline", }); nxpImxPipeline.addDependency(buildImagePipeline) + +/** + * Create an Embedded Linux Codebuild Project. + */ +const codeBuildActionsEnv = new EmbeddedLinuxCodebuildProjectStack(app, "EmbeddedLinuxCodeBuildProject", { + ...defaultProps, + imageRepo: buildImageRepo.repository, + imageTag: ImageKind.Ubuntu22_04, + vpc: vpc.vpc, + projectKind: ProjectKind.CodeBuild, +}); +codeBuildActionsEnv.addDependency(buildImagePipeline) diff --git a/package-lock.json b/package-lock.json index 3ed320a..0820767 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,16 @@ { "name": "meta-aws-cdk-pipeline-reference", - "version": "0.1.2", + "version": "0.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "meta-aws-cdk-pipeline-reference", - "version": "0.1.2", + "version": "0.1.4", "dependencies": { - "aws-cdk-lib": "2.86.0", + "aws-cdk-lib": "^2.170.0", "aws4embeddedlinux-cdk-lib": "github:aws4embeddedlinux/aws4embeddedlinux-ci", - "constructs": "^10.0.0", + "constructs": "^10.4.2", "source-map-support": "^0.5.21" }, "bin": { @@ -21,7 +21,7 @@ "@types/node": "20.1.0", "@typescript-eslint/eslint-plugin": "^5.59.6", "@typescript-eslint/parser": "^5.59.6", - "aws-cdk-lib": "2.86.0", + "aws-cdk-lib": "^2.170.0", "eslint": "^8.40.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", @@ -56,19 +56,51 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.200", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", - "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==" + "version": "2.2.220", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.220.tgz", + "integrity": "sha512-2eXZnnIgwWmXc7eRh8mRKPp6yHTKiQrLziRX/oVSfp4M6Jn2no0QFKJoHWqziF5MDQa5TF8qhD4FGsls/1nYPg==" }, "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", - "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.3.tgz", + "integrity": "sha512-cDG1w3ieM6eOT9mTefRuTypk95+oyD7P5X/wRltwmYxU7nZc3+076YEVS6vrjDKr3ADYbfn0lDKpfB1FBtO9CQ==" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==" + }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "39.2.2", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-39.2.2.tgz", + "integrity": "sha512-w6OTPy/WI4nyXDRjfuyXeHYJVkRkdam95fc4gf4Xk7O/sFey62BoxFS+HWLlET64FJzTgZAluIQrQAtxPyiS4g==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.6.3" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v5": { - "version": "2.0.166", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.166.tgz", - "integrity": "sha512-j0xnccpUQHXJKPgCwQcGGNu4lRiC1PptYfdxBIH1L4dRK91iBxtSQHESRQX+yB47oGLaF/WfNN/aF3WXwlhikg==" + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.6.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } }, "node_modules/@babel/code-frame": { "version": "7.22.13", @@ -1802,9 +1834,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.86.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.86.0.tgz", - "integrity": "sha512-76yZ2MawAGXLD3ox4FjhUIPmAMXteGKkeo3tPMthemusDCCkD2X6DBssXBHjB7r9GnrOMMf8JH5BGq2lOZ539g==", + "version": "2.176.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.176.0.tgz", + "integrity": "sha512-6Gs2kBaq4elQ4fNAOiCgbD9oOLx/heb/Lp4OVE6Uf7FulYW0DikWJXxR5GWJslTJ4/sCf3UU91q415fc0bruLg==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -1815,21 +1847,24 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.177", - "@aws-cdk/asset-kubectl-v20": "^2.1.1", - "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.148", + "@aws-cdk/asset-awscli-v1": "^2.2.208", + "@aws-cdk/asset-kubectl-v20": "^2.1.3", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-schema": "^39.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^11.1.1", - "ignore": "^5.2.4", + "fs-extra": "^11.2.0", + "ignore": "^5.3.2", "jsonschema": "^1.4.1", + "mime-types": "^2.1.35", "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.5.1", - "table": "^6.8.1", + "punycode": "^2.3.1", + "semver": "^7.6.3", + "table": "^6.8.2", "yaml": "1.10.2" }, "engines": { @@ -1845,14 +1880,14 @@ "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.12.0", + "version": "8.17.1", "inBundle": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -1942,8 +1977,13 @@ "inBundle": true, "license": "MIT" }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.0.3", + "inBundle": true, + "license": "BSD-3-Clause" + }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.2.0", "inBundle": true, "license": "MIT", "dependencies": { @@ -1961,7 +2001,7 @@ "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.2", "inBundle": true, "license": "MIT", "engines": { @@ -2005,15 +2045,23 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/lru-cache": { - "version": "6.0.0", + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", "inBundle": true, - "license": "ISC", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", "dependencies": { - "yallist": "^4.0.0" + "mime-db": "1.52.0" }, "engines": { - "node": ">=10" + "node": ">= 0.6" } }, "node_modules/aws-cdk-lib/node_modules/minimatch": { @@ -2028,7 +2076,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -2044,12 +2092,9 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.5.2", + "version": "7.6.3", "inBundle": true, "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, "bin": { "semver": "bin/semver.js" }, @@ -2098,7 +2143,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.8.1", + "version": "6.8.2", "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -2113,26 +2158,13 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "inBundle": true, "license": "MIT", "engines": { "node": ">= 10.0.0" } }, - "node_modules/aws-cdk-lib/node_modules/uri-js": { - "version": "4.4.1", - "inBundle": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/yallist": { - "version": "4.0.0", - "inBundle": true, - "license": "ISC" - }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", "inBundle": true, @@ -2142,8 +2174,8 @@ } }, "node_modules/aws4embeddedlinux-cdk-lib": { - "version": "0.1.3", - "resolved": "git+ssh://git@github.com/aws4embeddedlinux/aws4embeddedlinux-ci.git#45f22e421b3ec9b5f911d715a2058a1b1a2b5e96", + "version": "0.1.4", + "resolved": "git+ssh://git@github.com/aws4embeddedlinux/aws4embeddedlinux-ci.git#f0ddfc4a1d5c218cf13659249c7f43ed94289815", "peerDependencies": { "aws-cdk-lib": "^2.86.0", "constructs": "^10.0.0" @@ -2490,12 +2522,9 @@ "dev": true }, "node_modules/constructs": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.3.0.tgz", - "integrity": "sha512-vbK8i3rIb/xwZxSpTjz3SagHn1qq9BChLEfy5Hf6fB3/2eFbrwt2n9kHwQcS0CPTRBesreeAcsJfMq2229FnbQ==", - "engines": { - "node": ">= 16.14.0" - } + "version": "10.4.2", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.2.tgz", + "integrity": "sha512-wsNxBlAott2qg8Zv87q3eYZYgheb9lchtBfjHzzLHtXbttwSrHPs1NNQbBrmbb1YZvYg2+Vh0Dor76w4mFxJkA==" }, "node_modules/convert-source-map": { "version": "2.0.0", @@ -2531,9 +2560,9 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -4295,12 +4324,12 @@ } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { diff --git a/package.json b/package.json index 98381de..1ef65f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "meta-aws-cdk-pipeline-reference", - "version": "0.1.2", + "version": "0.1.4", "bin": { "embedded-linux-pipeline": "bin/app.js" }, @@ -19,7 +19,7 @@ "@types/node": "20.1.0", "@typescript-eslint/eslint-plugin": "^5.59.6", "@typescript-eslint/parser": "^5.59.6", - "aws-cdk-lib": "2.86.0", + "aws-cdk-lib": "^2.170.0", "eslint": "^8.40.0", "eslint-config-prettier": "^8.8.0", "eslint-plugin-prettier": "^4.2.1", @@ -31,9 +31,9 @@ "typescript": "~5.0.4" }, "dependencies": { - "aws-cdk-lib": "2.86.0", + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.4.2", "aws4embeddedlinux-cdk-lib": "github:aws4embeddedlinux/aws4embeddedlinux-ci", - "constructs": "^10.0.0", "source-map-support": "^0.5.21" } }