diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000000..6173a70be8 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,7 @@ +**/node_modules +**/coverage +test/fixtures +build/ +docs/ +protos/ +static/ \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000000..7e5a1dd078 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,17 @@ +{ + "root": true, + "extends": "./node_modules/gts/", + "env": { + "mocha": true + }, + "rules": { + "node/no-missing-import": ["off"], + "node/no-missing-require": ["off"], + "node/no-unpublished-import": ["off"], + "node/no-unpublished-require": ["off"], + "node/no-unsupported-features/es-syntax": ["off"] + }, + "parserOptions": { + "sourceType": "module" + } +} diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 0000000000..a231469f2d --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest + digest: sha256:fe04ae044dadf5ad88d979dbcc85e0e99372fb5d6316790341e6aca5e4e3fbc8 + diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 0000000000..b550fa110e --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest + +begin-after-commit-hash: db31e3ff07d737b61ce968625aabbf660501688c diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..e52d1cbebb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,30 @@ +--- +name: Bug report +about: Please let us know which issues you are having. +title: '' +labels: 'priority: p2, triage me, type: bug' +assignees: '' + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +**The issue you're having must be related to a file in this repository.** We are unable to provide assistance for issues unrelated to samples in this repository. + +Please include as much information as possible: + +## In which file did you encounter the issue? + + +## Did you change the file? If so, how? + + +## Describe the issue + + +Making sure to follow these steps will guarantee the quickest resolution possible. + +Thanks! \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..51f7678212 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Let us know how we can make things better. +title: '' +labels: 'priority: p3, triage me, type: feature request' +assignees: '' + +--- + +Thanks for stopping by to let us know something could be better! + +**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. + +**The issue you're having must be related to a file in this repository.** We are unable to provide assistance for issues unrelated to samples in this repository. + +Please include as much information as possible: + +## Is your feature request related to a problem? Please describe. + + +## Describe the solution you'd like. + + +## Describe alternatives you've considered. + + +## Additional context. + + +Making sure to follow these steps will guarantee the quickest resolution possible. + +Thanks! \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..b63c3f817f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +## Description + +Fixes # + +Note: Before submitting a pull request, please open an issue for discussion if you are not associated with Google. + +## Checklist +- [ ] I have followed guidelines from [CONTRIBUTING.MD](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md) and [Samples Style Guide](https://googlecloudplatform.github.io/samples-style-guide/) +- [ ] **Tests** pass: `npm test` (see [Testing](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md#run-the-tests-for-a-single-sample)) +- [ ] **Lint** pass: `npm run lint` (see [Style](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md#style)) +- [ ] **Required CI tests** pass (see [CI testing](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md#ci-testing)) +- [ ] These samples need a new **API enabled** in testing projects to pass (let us know which ones) +- [ ] These samples need a new/updated **env vars** in testing projects set to pass (let us know which ones) +- [ ] This pull request is from a branch created directly off of `GoogleCloudPlatform/nodejs-docs-samples`. Not a fork. +- [ ] This sample adds a new sample directory, and I updated the [CODEOWNERS file](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CODEOWNERS) with the codeowners for this sample +- [ ] This sample adds a new sample directory, and I created [GitHub Actions workflow](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md#adding-new-samples) for this sample +- [ ] This sample adds a new **Product API**, and I updated the [Blunderbuss issue/PR auto-assigner](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/.github/blunderbuss.yml) with the codeowners for this sample +- [ ] Please **merge** this PR for me once it is approved + +> **Note**: Any check with `(dev)`, `(experimental)`, or `(legacy)` can be ignored and should **not block** your PR from merging (see [CI testing](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/main/CONTRIBUTING.md#ci-testing)). diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml new file mode 100644 index 0000000000..01f6d2b337 --- /dev/null +++ b/.github/auto-label.yaml @@ -0,0 +1,79 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +path: + pullrequest: true + multipleLabelPaths: + - labelprefix: "api: " + paths: + ai-platform: "aiplatform" + appengine: "appengine" + asset: "cloudasset" + auth: "auth" + automl: "automl" + batch: "batch" + cloud-language: "language" + cloud-sql: "cloudsql" + cloud-tasks: "tasks" + cloudbuild: "cloudbuild" + composer: "composer" + compute: "compute" + contact-center-insights: "contactcenterinsights" + container: "container" + container-analysis: "containeranalysis" + datacatalog: "datacatalog" + datalabeling: "datalabeling" + dataproc: "dataproc" + datastore: "datastore" + dialogflow: "dialogflow" + dialogflow-cx: "dialogflow" + discoveryengine: "discoveryengine" + dlp: "dlp" + document-ai: "documentai" + endpoints: "endpoints" + eventarc: "eventarc" + error-reporting: "clouderrorreporting" + functions: "cloudfunctions" + generative-ai: "genai" + healthcare: "healhcare" + iam: "iam" + kms: "cloudkms" + mediatranslation: "mediatranslation" + memorystore: "memorystore" + monitoring: "monitoring" + recaptcha_enterprise: "recaptchaenterprise" + retail: "retail" + run: "run" + scheduler: "cloudscheduler" + secret-manager: "secretmanager" + security-center: "securitycenter" + service-directory: "servicedirectory" + speech: "speech" + talent: "jobs" + texttospeech: "texttospeech" + translate: "translate" + video-intelligence: "videointelligence" + vision: "vision" + workflows: "workflows" + - labelprefix: "asset: " + paths: + appengine: "pattern" + cloud-sql: "pattern" + cloud-tasks: + tutorial-gcf: "pattern" + eventarc: "pattern" + functions: "pattern" + recaptcha_enterprise: + demosite: "flagship" + run: "pattern" diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml new file mode 100644 index 0000000000..83cbed66df --- /dev/null +++ b/.github/blunderbuss.yml @@ -0,0 +1,87 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +assign_issues_by: +- labels: + - "api: asset" + to: + - GoogleCloudPlatform/cloud-asset-analysis-team +- labels: + - 'api: bigtable' + - 'api: datastore' + - 'api: firestore' + to: + - GoogleCloudPlatform/cloud-native-db-dpes +- labels: + - 'api: cloudsql' + to: + - GoogleCloudPlatform/cloud-sql-connectors +- labels: + - 'api: dlp' + to: + - GoogleCloudPlatform/googleapis-dlp +- labels: + - 'api: contentwarehouse' + - 'api: documentai' + to: + - GoogleCloudPlatform/document-ai-samples-contributors +- labels: + - "api: routeoptimization" + to: + - GoogleCloudPlatform/geo-routeoptimization +- labels: + - "api: parametermanager" + to: + - GoogleCloudPlatform/cloud-parameters-team +- labels: + - "api: modelarmor" + to: + - GoogleCloudPlatform/cloud-modelarmor-team + +assign_prs_by: +- labels: + - "api: asset" + to: + - GoogleCloudPlatform/cloud-asset-analysis-team +- labels: + - 'api: bigtable' + - 'api: datastore' + - 'api: firestore' + to: + - GoogleCloudPlatform/cloud-native-db-dpes +- labels: + - 'api: cloudsql' + to: + - GoogleCloudPlatform/cloud-sql-connectors +- labels: + - 'api: dlp' + to: + - GoogleCloudPlatform/googleapis-dlp +- labels: + - 'api: contentwarehouse' + - 'api: documentai' + to: + - GoogleCloudPlatform/document-ai-samples-contributors +- labels: + - "api: routeoptimization" + to: + - GoogleCloudPlatform/geo-routeoptimization +- labels: + - "api: parametermanager" + to: + - GoogleCloudPlatform/cloud-parameters-team +- labels: + - "api: modelarmor" + to: + - GoogleCloudPlatform/cloud-modelarmor-team diff --git a/.github/config/README.md b/.github/config/README.md new file mode 100644 index 0000000000..3e54a40de3 --- /dev/null +++ b/.github/config/README.md @@ -0,0 +1,273 @@ +# Node.js setup + +The CI setup file is an _optional_ file to configure the testing infrastructure runtime environment on a per-package basis. +It _must_ be named `ci-setup.json` and be located in the same directory as the `package.json` file. + +For example: + +```sh +my-product/ +└─ my-package/ + ├─ package.json # package file + └─ ci-setup.json # setup file +``` + +## Default values + +Here are the default values to be used if a `ci-setup.json` file is not present, or only some fields are defined. + +```js +{ + "env": {}, + "secrets": {}, + "node-version": 20, + "timeout-minutes": 10, +} +``` + +These are defined in the [`nodejs-prod.jsonc`](nodejs-prod.jsonc) config file. + +### Environment variables + +Environment variables are defined as a dictionary where the key is the environment variable name, and the value is the variable’s value. + +For example: + +```js +{ + "env": { + "MY_VAR": "my-value", + "MY_OTHER_VAR": "my-other-value" + } +} +``` + +The test infrastructure then exports them as environment variables. + +### Automatic environment variables + +Additionally to the environment variables you define in the `ci-setup.json` file, the test infrastructure always exports the following environment variables: + +- `PROJECT_ID`: The project used by the testing infrastructure. +- `RUN_ID`: A random unique identifier, different on every test run. +- `SERVICE_ACCOUNT`: The email of the testing service account. + +> **Note**: An environment variable explicitly defined under `env` with the same name as an automatic variable overrides the automatic variable. + +### Environment variable interpolation + +Environment variables support using variable interpolation in the format `$VAR` or `${VAR}`. +Secrets do not support this to avoid accidentally exposing secrets. + +This can be useful for fully qualified resource names like `projects/$PROJECT_ID/resources/my-resource-id`, or to create unique resource names like `my-resource-$RUN_ID`. + +## Unique resources + +If a test creates and destroys their own resources, make sure the resource names are different for each run. +This can be done by appending a unique identifier to each resource ID. We can use the `$RUN_ID` environment variable. + +For example: + +```js +{ + "env": { + "DATABASE": "projects/$PROJECT_ID/databases/my-db-$RUN_ID" + } +} +``` + +For more information, see +[Concurrent runs](https://github.com/GoogleCloudPlatform/cloud-samples-tools/blob/main/docs/testing-guidelines.md#concurrent-runs) +in our testing guidelines. + +### Secrets + +Secrets can be defined as a dictionary where the key is the environment variable to which it’s exported to, and the value is the +[Secret Manager](https://cloud.google.com/security/products/secret-manager) +secret ID in the format `project-id/secret-id`. + +For example: + +```js +{ + "secrets": { + "MY_SECRET": "$PROJECT_ID/my-secret-id", + "MY_OTHER_SECRET": "$PROJECT_ID/my-other-secret-id" + } +} +``` + +The test infrastructure then fetches the actual secret data and exports them as environment variables. + +### Automatic secret values + +Additional to any secret values you define in the `ci-setup.json` file, the test infrastructure always exports the following secret value: + +- `ID_TOKEN`: an ID token for interacting with private Cloud Run services. + + +
+ +Adapting `ID_TOKEN` use in system testing + + +Due to organization policies, Cloud Run services cannot be deployed with public +access. This means authentication is required in order to perform integration +testing. We do this by using ID Tokens (JWT) provided by [Google GitHub Actions +Auth](https://github.com/google-github-actions/auth/blob/main/docs/EXAMPLES.md#generating-an-id-token-jwt). + +By default, the audience of a Cloud Run service is [the full URL of the service +itself](https://cloud.google.com/run/docs/configuring/custom-audiences#:~:text=By%20convention%2C%20the%20audience%20is). +Since we cannot know all the URLs for all samples ahead of time (since unique +IDs are in use), we instead define a custom audience in the GitHub Action, then +apply that as an additional audience to all Cloud Run services. + +To use this method, some changes are required: + +1. As part of testing setup, add a step to customize the Cloud Run service to have the custom audience `https://action.test/` + + ```shell + gcloud run services deploy ${_SERVICE} \ + ... \ + --add-custom-audiences="https://action.test/" + ``` + +1. Use the environment variable ID_TOKEN in any Authorization: Bearer calls. + + For example, a curl command: + + ```shell + # ❌ Previous version: calls gcloud + curl -H "Authorization: Bearer $(gcloud auth print-identity-token)" https://my-service-hash.a.run.app + + # ✅ New version: uses environment variable + curl -H "Authorization: Bearer ${ID_TOKEN}" https://my-service-hash.a.run.app + ``` + + For example, a Node.JS script: + + ```javascript + // ❌ Previous version: auth.getIdTokenClient() + const client = await auth.getIdTokenClient(BASE_URL); + const clientHeaders = await client.getRequestHeaders(); + ID_TOKEN = clientHeaders['Authorization'].trim(); + if (!ID_TOKEN) throw Error('Unable to acquire an ID token.'); + + // ✅ New version: uses environment variable + {ID_TOKEN} = process.env; + if (!ID_TOKEN) throw Error('"ID_TOKEN" not found in environment variables.'); + ``` + +
+ +### Secrets vs environment variables + +Most configuration should be directly set with environment variables. +This includes some things which might appear to be sensitive, but aren't. +Minimizing secrets helps reduce maintenance and troubleshooting efforts. + +For example, when a secret gets exposed, we must go through all the secrets used on that repository, assess which ones are real secrets and rotate them, and provide a justificatin of secrets that weren't rotated because they're just configurations. +Keeping configurations as environment variables reduces the number of secrets needed to be assessed. + +Examples of environment variables: + +- Project IDs +- Cloud resource IDs (or ID templates for a set of tests) +- Service accounts emails (they look like email addresses, but aren't) +- Usernames and passwords for _private_ resources (meaning anyone without project permissions cannot access those resources, so they can’t even reach the login) + +These items are also likely to appear in logs, and that is OK. + +Some things really are secret and if exposed, would create opportunities for unauthorized access, fraud, etc. +The test infrastructure uses [Google Cloud Secret Manager](https://cloud.google.com/security/products/secret-manager) to store and retrieve secrets. + +Examples of real secrets: + +- API keys +- Auth tokens +- Identity tokens +- Service account credentials (the JSON content with the actual keys) +- Passwords or API keys to 3rd party services needed for testing +- Usernames and passwords for _public_ resources (accessible to anyone) + +> **Important**: Test code and scripts should _**never log secret values**_. +> A test run printing a secret will stay in the commit history, even if another commit is made after. +> This requires rotating that secret. + +> **Note**: These testing environment variables and secrets are completely separate from [GitHub Actions secrets](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions) and variables. +> GitHub Actions secrets and variables are _not_ used by the testing infrastructure. + +### Node version + +You can opt in to use a different Node version if you need a specific version functionality. + +For example: + +```js +{ + "_justification": "My reason for using a specific Node version.", + "node-version": 18, +} +``` + +We **strongly recommend** not overriding this unless it's a version-specific feature or test. + +This _might_ raise flags during review. + +If you have a valid reason to use a different Node version, please provide a justification as a `_comment`. + +> **Note**: In the future, we might implement automatic checks to validate that tests are using the recommended default version. + +### Test timeout + +If your tests take longer than the default test timeout, you can opt in to override it. + +For example: + +```js +{ + "_justification": "My reason for this test to take longer to run.", + "timeout-minutes": 30, +} +``` + +> **Note**: Overriding this **will** raise flags during review. + +Tests should run fast, and the default timeout should be more than enough for most tests to run. +If your tests are taking longer, consider splitting them into subpackages. + +For example: + +```sh +my-product/ +├─ snippets/ +│ ├─ package.json +│ ├─ fast-samples.js +│ ├─ test/ +│ │ └─ fast-samples.test.js +│ └─ slow-snippet/ +│ ├─ package.json +│ ├─ slow-snippet.js +│ └─ test/ +│ └─ slow-snippet.test.js +├─ new-feature/ +│ ├─ package.json +│ ├─ new-feature.js +│ └─ test/ +│ └─ new-feature.test.js +└─ e2e-sample/ + ├─ package.json + ├─ slow-sample.js + └─ test/ + └─ slow-sample.test.js +``` + +For more information, see +[Keep tests fast](https://github.com/GoogleCloudPlatform/cloud-samples-tools/blob/main/docs/testing-guidelines.md#keep-tests-fast) +in our testing guidelines. + +If you have a valid reason to use a different timeout, please provide a justification as a `_comment`. + +There is a hard limit of 120 minutes that cannot be overriden. + +> **Note**: In the future, we might implement automatic checks to validate that tests run fast to safely reduce the default timeout. diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc new file mode 100644 index 0000000000..c8ca78b18b --- /dev/null +++ b/.github/config/nodejs-dev.jsonc @@ -0,0 +1,228 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ + "package-file": [ "package.json" ], + "ci-setup-filename": "ci-setup.json", + + // If these change, please update the .github/config/README.md too! + "ci-setup-defaults": { + "env": { }, + "secrets": { }, + "node-version": 20, + "timeout-minutes": 10 + }, + + "ignore": [ + // TODO: do not ignore .github/config once everything is in prod + ".github/config/", // prevent changes to exclusions from running all tests + ".eslintignore", + ".eslintrc.json", + ".github/.OwlBot.lock.yaml", + ".github/.OwlBot.yaml", + ".github/ISSUE_TEMPLATE/", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/auto-label.yaml", + ".github/blunderbuss.yml", + ".github/flakybot.yaml", + ".github/header-checker-lint.yml", + ".github/scripts/", + ".github/snippet-bot.yml", + ".github/trusted-contribution.yml", + // TODO: do not ignore .github/workflows once everything is in prod + ".github/workflows/", // prevent removing old tests to trigger everything + ".gitignore", + ".kokoro/", + ".prettierignore", + ".prettierrc.js", + "cloud-samples-tools", // checked out by GH action in ci-*.yml + "CODEOWNERS", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "Makefile", + "README.md", + "SECURITY.md", + "buildsetup.sh", + "linkinator.config.json", + "node_modules/", + "owlbot.py", + "renovate.json" + ], + + // These are all working well in prod, so we can exclude them from dev. + // Once all packages are in prod, we can remove this dev config. + "exclude-packages": [ + "functions/concepts", // parent directory + "functions/firebase", // parent directory + "functions/helloworld", // parent directory + "functions/http", // parent directory + "functions/http/uploadFile", // no tests exist + "functions/log", // parent directory + "functions/pubsub", // parent directory + "memorystore/redis", // parent directory + "recaptcha_enterprise/demosite/app", // no tests exist + + // These tests are already passing in prod, so skip them in dev. + "appengine/building-an-app/build", + "appengine/building-an-app/update", + "appengine/datastore", + "appengine/endpoints", + "appengine/hello-world/flexible", + "appengine/hello-world/flexible_nodejs16_and_earlier", + "appengine/hello-world/standard", + "appengine/memcached", + "appengine/analytics", + "appengine/metadata/flexible", + "appengine/metadata/flexible_nodejs16_and_earlier", + "appengine/metadata/standard", + "appengine/pubsub", + "appengine/static-files", + "appengine/storage/flexible", + "appengine/storage/flexible_nodejs16_and_earlier", + "appengine/storage/standard", + "appengine/typescript", + "appengine/websockets", + "asset/snippets", + "auth", + "batch", + "cloud-language", + "cloud-sql/mysql/mysql", + "cloud-sql/mysql/mysql2", + "cloud-sql/postgres/knex", + "cloud-sql/sqlserver/mssql", + "cloud-tasks/snippets", + "cloud-tasks/tutorial-gcf/app", + "cloud-tasks/tutorial-gcf/function", + "cloudbuild", + "composer", + "composer/functions/composer-storage-trigger", + "contact-center-insights", + "container", + "container-analysis/snippets", + "datacatalog/cloud-client", + "datacatalog/quickstart", + "datacatalog/snippets", + "datalabeling", + "dialogflow", + "discoveryengine", + "document-warehouse", + "endpoints/getting-started", + "endpoints/getting-started-grpc", + "error-reporting", + "eventarc/audit-storage", + "eventarc/generic", + "eventarc/pubsub", + "functions/concepts/afterResponse", + "functions/concepts/afterTimeout", + "functions/concepts/backgroundTermination", + "functions/concepts/filesystem", + "functions/concepts/httpTermination", + "functions/concepts/requests", + "functions/concepts/stateless", + "functions/env_vars", + "functions/firebase/helloAnalytics", + "functions/firebase/helloAuth", + "functions/firebase/helloFirestore", + "functions/firebase/helloRTDB", + "functions/firebase/helloRemoteConfig", + "functions/firebase/makeUpperCase", + "functions/helloworld/helloError", + "functions/helloworld/helloGCS", + "functions/helloworld/helloPubSub", + "functions/helloworld/helloworldGet", + "functions/helloworld/helloworldHttp", + "functions/http/corsEnabledFunction", + "functions/http/corsEnabledFunctionAuth", + "functions/http/httpContent", + "functions/http/httpMethods", + "functions/http/parseXML", + "functions/imagemagick", + "functions/log/helloWorld", + "functions/log/processEntry", + "functions/memorystore/redis", + "functions/ocr/app", + "functions/pubsub/publish", + "functions/pubsub/subscribe", + "functions/scheduleinstance", + "functions/security", + "functions/spanner", + "functions/tips", + "functions/tips/avoidInfiniteRetries", + "functions/tips/connectionPools", + "functions/tips/gcpApiCall", + "functions/tips/lazyGlobals", + "functions/tips/retry", + "functions/tips/scopeDemo", + "functions/v2/autoLabelInstance", + "functions/v2/cloudEventLogging", + "functions/v2/firebase/firestore/helloFirestore", + "functions/v2/firebase/firestore/makeUpperCase", + "functions/v2/firebase/remote-config/helloRemoteConfig", + "functions/v2/firebase/rtdb/helloRTDB", + "functions/v2/helloAuditLog", + "functions/v2/helloBigQuery", + "functions/v2/helloGCS", + "functions/v2/helloPubSub", + "functions/v2/httpLogging", + "functions/v2/imagemagick", + "functions/v2/log/processEntry", + "functions/v2/ocr/app", + "functions/v2/responseStreaming", + "functions/v2/tips/avoidInfiniteRetries", + "functions/v2/tips/retry", + "functions/v2/typed/googlechatbot", + "functions/v2/typed/greeting", + "generative-ai/snippets", + "healthcare/consent", + "healthcare/datasets", + "healthcare/dicom", + "healthcare/hl7v2", + "kms", + "media/livestream", + "media/transcoder", + "media/video-stitcher", + "mediatranslation", + "monitoring/prometheus", + "monitoring/snippets", + "recaptcha_enterprise/snippets", + "retail", + "routeoptimization/snippets", + "run/hello-broken", + "run/helloworld", + "run/image-processing", + "run/jobs", + "run/logging-manual", + "run/idp-sql", + "run/markdown-preview/editor", + "run/markdown-preview/renderer", + "run/pubsub", + "run/system-package", + "run/websockets", + "secret-manager", + "security-center/snippets", + "service-directory/snippets", + "scheduler", + "storage-control", + "speech", + "talent", + "texttospeech", + "tpu", + "translate", + "vision", + "workflows/invoke-private-endpoint" + ] +} diff --git a/.github/config/nodejs.jsonc b/.github/config/nodejs.jsonc new file mode 100644 index 0000000000..b8bc67ffcb --- /dev/null +++ b/.github/config/nodejs.jsonc @@ -0,0 +1,96 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ + "package-file": [ "package.json" ], + "ci-setup-filename": "ci-setup.json", + + // If these change, please update the .github/config/README.md too! + "ci-setup-defaults": { + "env": { }, + "secrets": { }, + "node-version": 20, + "timeout-minutes": 10 + }, + + "ignore": [ + // TODO: do not ignore .github/config once everything is in prod + ".github/config/", // prevent changes to exclusions from running all tests + ".eslintignore", + ".eslintrc.json", + ".github/.OwlBot.lock.yaml", + ".github/.OwlBot.yaml", + ".github/ISSUE_TEMPLATE/", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/auto-label.yaml", + ".github/blunderbuss.yml", + ".github/flakybot.yaml", + ".github/header-checker-lint.yml", + ".github/scripts/", + ".github/snippet-bot.yml", + ".github/trusted-contribution.yml", + // TODO: do not ignore .github/workflows once everything is in prod + ".github/workflows/", // prevent removing old tests to trigger everything + ".gitignore", + ".kokoro/", + ".prettierignore", + ".prettierrc.js", + "cloud-samples-tools", // checked out by GH action in ci-*.yml + "CODEOWNERS", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "Makefile", + "README.md", + "SECURITY.md", + "buildsetup.sh", + "linkinator.config.json", + "node_modules/", + "owlbot.py", + "renovate.json" + ], + + "exclude-packages": [ + "functions/concepts", // parent directory + "functions/firebase", // parent directory + "functions/helloworld", // parent directory + "functions/http", // parent directory + "functions/http/uploadFile", // no tests exist + "functions/log", // parent directory + "functions/pubsub", // parent directory + "memorystore/redis", // parent directory + "recaptcha_enterprise/demosite/app", // no tests exist + + // TODO: fix these + "ai-platform/snippets", // PERMISSION_DENIED: Permission denied: Consumer 'projects/undefined' has been suspended. + "automl", // (untested) FAILED_PRECONDITION: Google Cloud AutoML Natural Language was retired on March 15, 2024. Please migrate to Vertex AI instead + "cloud-sql/sqlserver/tedious", // (untested) TypeError: The "config.server" property is required and must be of type string. + "compute", // GoogleError: The resource 'projects/long-door-651/zones/us-central1-a/disks/disk-from-pool-name' was not found + "dataproc", // GoogleError: Error submitting create cluster request: Multiple validation errors + "datastore/functions", // [ERR_REQUIRE_ESM]: require() of ES Module + "dialogflow-cx", // NOT_FOUND: com.google.apps.framework.request.NotFoundException: Agent 'undefined' does not exist + "dlp", // [ERR_REQUIRE_ESM]: require() of ES Module + "document-ai", // [ERR_REQUIRE_ESM]: require() of ES Module + "functions/billing", // (untested) Error: Request failed with status code 500 + "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined + "healthcare/fhir", // Error: Cannot find module 'whatwg-url' + "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 + "storagetransfer", // CredentialsError: Missing credentials in config, if using AWS_CONFIG_FILE, set AWS_SDK_LOAD_CONFIG=1 + "video-intelligence", // PERMISSION_DENIED: The caller does not have permission + "workflows", // SyntaxError: Cannot use import statement outside a module + "workflows/quickstart" // [ERR_MODULE_NOT_FOUND]: Cannot find package 'ts-node' imported from ... + ] +} diff --git a/.github/flakybot.yaml b/.github/flakybot.yaml new file mode 100644 index 0000000000..9bc86c4f1d --- /dev/null +++ b/.github/flakybot.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +issuePriority: p2 diff --git a/.github/header-checker-lint.yml b/.github/header-checker-lint.yml new file mode 100644 index 0000000000..90a15de6b9 --- /dev/null +++ b/.github/header-checker-lint.yml @@ -0,0 +1,29 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +allowedCopyrightHolders: + - 'Google LLC' + - 'Google, Inc' +allowedLicenses: + - 'Apache-2.0' + - 'MIT' + - 'BSD-3' +sourceFileExtensions: + - 'ts' + - 'js' + - 'html' + - 'yaml' + - 'yml' + - 'sh' +ignoreLicenseYear: true diff --git a/.github/scripts/cli/vars.js b/.github/scripts/cli/vars.js new file mode 100644 index 0000000000..6ab2532cc3 --- /dev/null +++ b/.github/scripts/cli/vars.js @@ -0,0 +1,41 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import setupVars from '../setup-vars.js'; + +const project_id = process.env.PROJECT_ID; +if (!project_id) { + console.error( + 'Please set the PROJECT_ID environment variable to your Google Cloud project.' + ); + process.exit(1); +} + +const core = { + exportVariable: (_key, _value) => null, +}; + +const setupFile = process.argv[2]; +if (!setupFile) { + console.error('Please provide the path to a setup file.'); + process.exit(1); +} +const data = fs.readFileSync(path.join('..', '..', setupFile), 'utf8'); +const setup = JSON.parse(data); + +setupVars({project_id, core, setup}); diff --git a/.github/scripts/package.json b/.github/scripts/package.json new file mode 100644 index 0000000000..97a284ad97 --- /dev/null +++ b/.github/scripts/package.json @@ -0,0 +1,14 @@ +{ + "name": "aritest", + "version": "1.0.0", + "type": "module", + "license": "Apache-2.0", + "private": true, + "scripts": { + "vars": "node cli/vars.js", + "test": "mocha -p -j 2 **/*.test.js" + }, + "devDependencies": { + "mocha": "^11.1.0" + } +} diff --git a/.github/scripts/setup-vars.js b/.github/scripts/setup-vars.js new file mode 100644 index 0000000000..54202abcca --- /dev/null +++ b/.github/scripts/setup-vars.js @@ -0,0 +1,81 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +export default function setupVars({projectId, core, setup, serviceAccount, idToken}, runId = null) { + // Define automatic variables plus custom variables. + const vars = { + PROJECT_ID: projectId, + RUN_ID: runId || uniqueId(), + SERVICE_ACCOUNT: serviceAccount, + ...(setup.env || {}), + }; + + // Apply variable interpolation. + const env = Object.fromEntries( + Object.keys(vars).map(key => [key, substituteVars(vars[key], vars)]) + ); + + // Export environment variables. + console.log('env:'); + for (const key in env) { + const value = env[key]; + console.log(` ${key}: ${value}`); + core.exportVariable(key, value); + } + + // Show exported secrets, for logging purposes. + // TODO: We might want to fetch the secrets here and export them directly. + // https://cloud.google.com/secret-manager/docs/create-secret-quickstart#secretmanager-quickstart-nodejs + console.log('secrets:'); + for (const key in setup.secrets || {}) { + // This is the Google Cloud Secret Manager secret ID. + // NOT the secret value, so it's ok to show. + console.log(` ${key}: ${setup.secrets[key]}`); + } + + // Set global secret for the Service Account identity token + // Use in place of 'gcloud auth print-identity-token' or auth.getIdTokenClient + // usage: curl -H 'Bearer: $ID_TOKEN' https:// + core.exportVariable('ID_TOKEN', idToken) + core.setSecret(idToken) + // For logging, show the source of the ID_TOKEN + console.log(` ID_TOKEN: steps.auth.outputs.id_token (from GitHub Action)`); + + // Return env and secrets to use for further steps. + return { + env: env, + // Transform secrets into the format needed for the GHA secret manager step. + secrets: Object.keys(setup.secrets || {}) + .map(key => `${key}:${setup.secrets[key]}`) + .join('\n'), + }; +} + +export function substituteVars(value, env) { + for (const key in env) { + let re = new RegExp(`\\$(${key}\\b|\\{\\s*${key}\\s*\\})`, 'g'); + value = value.replaceAll(re, env[key]); + } + return value; +} + +export function uniqueId(length = 6) { + const min = 2 ** 32; + const max = 2 ** 64; + return Math.floor(Math.random() * max + min) + .toString(36) + .slice(0, length); +} diff --git a/.github/scripts/setup-vars.test.js b/.github/scripts/setup-vars.test.js new file mode 100644 index 0000000000..e1f4b47b90 --- /dev/null +++ b/.github/scripts/setup-vars.test.js @@ -0,0 +1,156 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import {deepStrictEqual} from 'assert'; +import setupVars from './setup-vars.js'; +import {substituteVars, uniqueId} from './setup-vars.js'; + +const projectId = 'my-test-project'; +const serviceAccount = "my-sa@my-project.iam.gserviceaccount.com" +const core = { + exportVariable: (_key, _value) => null, + setSecret: (_key) => null, +}; + +const autovars = {PROJECT_ID: projectId, RUN_ID: 'run-id', SERVICE_ACCOUNT: serviceAccount}; + +describe('setupVars', () => { + describe('env', () => { + it('empty', () => { + const setup = {}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = autovars; + deepStrictEqual(vars.env, expected); + }); + + it('zero vars', () => { + const setup = {env: {}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = autovars; + deepStrictEqual(vars.env, expected); + }); + + it('one var', () => { + const setup = {env: {A: 'x'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = {...autovars, A: 'x'}; + deepStrictEqual(vars.env, expected); + }); + + it('three vars', () => { + const setup = {env: {A: 'x', B: 'y', C: 'z'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = {...autovars, A: 'x', B: 'y', C: 'z'}; + deepStrictEqual(vars.env, expected); + }); + + it('should override automatic variables', () => { + const setup = {env: {PROJECT_ID: 'custom-value', SERVICE_ACCOUNT: 'baz@foo.com'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = {PROJECT_ID: 'custom-value', RUN_ID: 'run-id', SERVICE_ACCOUNT: 'baz@foo.com'}; + deepStrictEqual(vars.env, expected); + }); + + it('should interpolate variables', () => { + const setup = {env: {A: 'x', B: 'y', C: '$A/${B}'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = {...autovars, A: 'x', B: 'y', C: 'x/y'}; + deepStrictEqual(vars.env, expected); + }); + + it('should not interpolate secrets', () => { + const setup = { + env: {C: '$x/$y'}, + secrets: {A: 'x', B: 'y'}, + }; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = {...autovars, C: '$x/$y'}; + deepStrictEqual(vars.env, expected); + }); + }); + + describe('secrets', () => { + it('zero secrets', () => { + const setup = {secrets: {}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + deepStrictEqual(vars.secrets, ''); + }); + + it('one secret', () => { + const setup = {secrets: {A: 'x'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = 'A:x'; + deepStrictEqual(vars.secrets, expected); + }); + + it('three secrets', () => { + const setup = {secrets: {A: 'x', B: 'y', C: 'z'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = 'A:x\nB:y\nC:z'; + deepStrictEqual(vars.secrets, expected); + }); + + it('should not interpolate variables', () => { + const setup = { + env: {A: 'x', B: 'y'}, + secrets: {C: '$A/$B'}, + }; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = 'C:$A/$B'; + deepStrictEqual(vars.secrets, expected); + }); + + it('should not interpolate secrets', () => { + const setup = {secrets: {A: 'x', B: 'y', C: '$A/$B'}}; + const vars = setupVars({projectId, core, setup, serviceAccount}, 'run-id'); + const expected = 'A:x\nB:y\nC:$A/$B'; + deepStrictEqual(vars.secrets, expected); + }); + }); +}); + +describe('substituteVars', () => { + it('should interpolate $VAR', () => { + const got = substituteVars('$A-$B', {A: 'x', B: 'y'}); + const expected = 'x-y'; + deepStrictEqual(got, expected); + }); + + it('should interpolate ${VAR}', () => { + const got = substituteVars('${A}-${B}', {A: 'x', B: 'y'}); + const expected = 'x-y'; + deepStrictEqual(got, expected); + }); + + it('should interpolate ${ VAR }', () => { + const got = substituteVars('${ A }-${ \tB\t }', {A: 'x', B: 'y'}); + const expected = 'x-y'; + deepStrictEqual(got, expected); + }); + + it('should not interpolate on non-word boundary', () => { + const got = substituteVars('$Ab', {A: 'x'}); + const expected = '$Ab'; + deepStrictEqual(got, expected); + }); +}); + +describe('uniqueId', () => { + it('should match length', () => { + const n = 6; + deepStrictEqual(uniqueId(n).length, n); + }); +}); diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/.github/snippet-bot.yml @@ -0,0 +1 @@ + diff --git a/.github/trusted-contribution.yml b/.github/trusted-contribution.yml new file mode 100644 index 0000000000..84f0d27f6e --- /dev/null +++ b/.github/trusted-contribution.yml @@ -0,0 +1,7 @@ +annotations: + - type: comment + text: "/gcbrun" + - type: label + text: "kokoro:force-run" + - type: label + text: "actions:force-run" diff --git a/.github/workflows/ai-platform-snippets.yaml b/.github/workflows/ai-platform-snippets.yaml new file mode 100644 index 0000000000..8fd3cd670d --- /dev/null +++ b/.github/workflows/ai-platform-snippets.yaml @@ -0,0 +1,98 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: ai-platform-snippets +on: + push: + branches: + - main + paths: + - 'ai-platform/snippets/**' + - '.github/workflows/ai-platform-snippets.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'ai-platform/snippets/**' + - '.github/workflows/ai-platform-snippets.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce' # v2 + with: + secrets: |- + caip_id:nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-caip-project-id + location:nodejs-docs-samples-tests/nodejs-docs-samples-ai-platform-location + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: "${{ runner.os }}-node- \n" + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=ai-platform-snippets" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - name: Run Tests + run: make test dir=ai-platform/snippets + env: + GOOGLE_SAMPLES_PROJECT: "long-door-651" + LOCATION: ${{ steps.secrets.outputs.location }} + CAIP_PROJECT_ID: ${{ steps.secrets.outputs.caip_id }} + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: ai-platform/snippets/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/ci-scripts.yaml b/.github/workflows/ci-scripts.yaml new file mode 100644 index 0000000000..6417fcbd44 --- /dev/null +++ b/.github/workflows/ci-scripts.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: scripts +on: + push: + branches: + - main + paths: + - .github/scripts/** + pull_request: + paths: + - .github/scripts/** + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: .github/scripts + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + - run: npm install + - run: npm test diff --git a/.github/workflows/compute.yaml b/.github/workflows/compute.yaml new file mode 100644 index 0000000000..ceb504fe8e --- /dev/null +++ b/.github/workflows/compute.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: compute +on: + push: + branches: + - main + paths: + - 'compute/**' + - '.github/workflows/compute.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'compute/**' + - '.github/workflows/compute.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'compute' + path: 'compute' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/custard-ci-dev.yaml b/.github/workflows/custard-ci-dev.yaml new file mode 100644 index 0000000000..c987d56718 --- /dev/null +++ b/.github/workflows/custard-ci-dev.yaml @@ -0,0 +1,117 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: (legacy dev) Custard CI +on: + push: + branches: + - main + pull_request: + # schedule: + # # https://crontab.guru/#0_12_*_*_0 + # - cron: 0 12 * * 0 # At 12:00 on Sunday + +env: + GO_VERSION: ^1.22.0 + +jobs: + affected: + name: (legacy) Finding affected tests + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + nodejs-paths: ${{ steps.nodejs.outputs.paths }} + nodejs-setups: ${{ steps.nodejs.outputs.setups }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: GoogleCloudPlatform/cloud-samples-tools + ref: v0.3.2 + path: cloud-samples-tools + - name: Create `bin` directory for cloud-samples-tools binaries + run: mkdir bin + working-directory: cloud-samples-tools + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Build Custard (from cloud-samples-tools) + run: go build -o ../bin -v ./... + working-directory: cloud-samples-tools/custard + - name: Get diffs + run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt + - name: Find Node.js affected packages + id: nodejs + run: | + echo "paths=$(./cloud-samples-tools/bin/custard affected .github/config/nodejs-dev.jsonc diffs.txt paths.txt)" >> $GITHUB_OUTPUT + cat paths.txt + echo "setups=$(./cloud-samples-tools/bin/custard setup-files .github/config/nodejs-dev.jsonc paths.txt)" >> $GITHUB_OUTPUT + + nodejs-test: + if: github.event.pull_request && github.event.pull_request.head.repo.fork == false + name: (legacy) test + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit + permissions: + id-token: write # needed for google-github-actions/auth + strategy: + fail-fast: false + matrix: + path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }} + env: + GOOGLE_SAMPLES_PROJECT: long-door-651 + GOOGLE_SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + CI_SETUP: ${{ toJson(fromJson(needs.affected.outputs.nodejs-setups)[matrix.path])}} + steps: + - name: CI Setup + run: echo "${{ env.CI_SETUP }}" + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ fromJson(env.CI_SETUP).node-version }} + - uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2 + id: auth + with: + project_id: ${{ env.GOOGLE_SAMPLES_PROJECT }} + workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider + service_account: ${{ env.GOOGLE_SERVICE_ACCOUNT }} + access_token_lifetime: 600s # 10 minutes + token_format: 'id_token' + id_token_audience: 'https://action.test/' # service must have this custom audience + id_token_include_email: true + - name: Export environment variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: vars + with: + script: | + const { default: setupVars } = await import('${{ github.workspace }}/.github/scripts/setup-vars.js'); + const projectId = '${{ env.GOOGLE_SAMPLES_PROJECT }}'; + const setup = JSON.parse(process.env.CI_SETUP); + const serviceAccount = '${{ env.GOOGLE_SERVICE_ACCOUNT }}'; + const idToken = '${{ steps.auth.outputs.id_token }}'; + return await setupVars({projectId, core, setup, serviceAccount, idToken}) + - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2 + if: ${{ fromJson(steps.vars.outputs.result).secrets }} + with: + secrets: ${{ fromJson(steps.vars.outputs.result).secrets }} + export_to_environment: true + - name: Run tests for ${{ matrix.path }} + run: | + timeout ${{ fromJson(env.CI_SETUP).timeout-minutes }}m \ + make test dir=${{ matrix.path }} diff --git a/.github/workflows/custard-ci.yaml b/.github/workflows/custard-ci.yaml new file mode 100644 index 0000000000..149f198a12 --- /dev/null +++ b/.github/workflows/custard-ci.yaml @@ -0,0 +1,185 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# IMPORTANT: DO NOT CHANGE THE NAME OF THIS WORKFLOW! +# The workflow named "Custard CI" triggers both custard-run.yaml and custard-run-dev.yaml. +# TODO: To sunset old tests remove `affected`, `lint`, and `test` jobs (only keep `region-tags`). +name: Custard CI +on: + push: + branches: + - main + pull_request: + # schedule: + # # https://crontab.guru/#0_12_*_*_0 + # - cron: 0 12 * * 0 # At 12:00 on Sunday + +env: + GO_VERSION: ^1.22.0 + +# TODO: remove all jobs except region-tags (should be tested by custard-run workflows) +jobs: + affected: + name: (legacy) Finding affected tests + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + nodejs-paths: ${{ steps.nodejs.outputs.paths }} + nodejs-setups: ${{ steps.nodejs.outputs.setups }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: GoogleCloudPlatform/cloud-samples-tools + ref: v0.3.2 + path: cloud-samples-tools + - name: Create `bin` directory for cloud-samples-tools binaries + run: mkdir bin + working-directory: cloud-samples-tools + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Build Custard (from cloud-samples-tools) + run: go build -o ../bin -v ./... + working-directory: cloud-samples-tools/custard + - name: Get diffs + run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt + - name: Find Node.js affected packages + id: nodejs + run: | + echo "paths=$(./cloud-samples-tools/bin/custard affected .github/config/nodejs.jsonc diffs.txt paths.txt)" >> $GITHUB_OUTPUT + cat paths.txt + echo "setups=$(./cloud-samples-tools/bin/custard setup-files .github/config/nodejs.jsonc paths.txt)" >> $GITHUB_OUTPUT + + lint: + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - name: Setup Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + - run: npm install + - name: Run lint + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + with: + script: | + const { execSync } = await import("node:child_process"); + + const cmd = 'npx gts lint'; + const affected = ${{ needs.affected.outputs.nodejs-paths }}; + if (affected.length === 0) { + console.log("No packages were affected, nothing to lint.") + } + + let failed = []; + for (const path of affected) { + try { + execSync(cmd, {cwd: path}); + console.log(`✅ [${path}]: ${cmd}`); + } catch (e) { + failed.push(path) + console.log(`❌ [${path}]: ${cmd} (exit code ${e.status})`); + core.error(e.message); + console.log('--- stdout ---'); + console.log(e.stdout.toString("utf8")); + console.log('--- stderr ---'); + console.log(e.stderr.toString("utf8")); + } + } + console.log("=== Summary ===") + console.log(` Passed: ${affected.length - failed.length}`) + console.log(` Failed: ${failed.length}`) + if (failed.length > 0) { + core.setFailed(`Failed '${cmd}' on: ${failed.join(', ')}`) + } + + region-tags: + name: region tags + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + - run: ./.github/workflows/utils/region-tags-tests.sh + + test: + if: github.event.pull_request && github.event.pull_request.head.repo.fork == false + name: (legacy) test + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit + permissions: + id-token: write # needed for google-github-actions/auth + strategy: + fail-fast: false + matrix: + path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }} + env: + GOOGLE_SAMPLES_PROJECT: long-door-651 + GOOGLE_SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + CI_SETUP: ${{ toJson(fromJson(needs.affected.outputs.nodejs-setups)[matrix.path])}} + steps: + - name: CI Setup + run: echo "${{ env.CI_SETUP }}" + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ fromJson(env.CI_SETUP).node-version }} + - uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2 + id: auth + with: + project_id: ${{ env.GOOGLE_SAMPLES_PROJECT }} + workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider + service_account: ${{ env.GOOGLE_SERVICE_ACCOUNT }} + access_token_lifetime: 600s # 10 minutes + token_format: 'id_token' + id_token_audience: 'https://action.test/' # service must have this custom audience + id_token_include_email: true + - name: Export environment variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: vars + with: + script: | + const { default: setupVars } = await import('${{ github.workspace }}/.github/scripts/setup-vars.js'); + const projectId = '${{ env.GOOGLE_SAMPLES_PROJECT }}'; + const setup = JSON.parse(process.env.CI_SETUP); + const serviceAccount = '${{ env.GOOGLE_SERVICE_ACCOUNT }}'; + const idToken = '${{ steps.auth.outputs.id_token }}'; + return await setupVars({projectId, core, setup, serviceAccount, idToken}) + - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2 + if: ${{ fromJson(steps.vars.outputs.result).secrets }} + with: + secrets: ${{ fromJson(steps.vars.outputs.result).secrets }} + export_to_environment: true + - name: Run tests for ${{ matrix.path }} + run: | + timeout ${{ fromJson(env.CI_SETUP).timeout-minutes }}m \ + make test dir=${{ matrix.path }} + # - name: Upload test results for FlakyBot workflow + # if: github.event.action == 'schedule' && always() # always() submits logs even if tests fail + # uses: actions/upload-artifact@v4 + # with: + # name: test-results + # path: ${{ matrix.package }}/${{ env.MOCHA_REPORTER_OUTPUT }} + # retention-days: 1 diff --git a/.github/workflows/custard-run-dev.yaml b/.github/workflows/custard-run-dev.yaml new file mode 100644 index 0000000000..a7f9957ae9 --- /dev/null +++ b/.github/workflows/custard-run-dev.yaml @@ -0,0 +1,140 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: (dev) Custard run + +on: + # Run tests when a pull request is created or updated. + # This allows to run tests from forked repos (after reviewer's approval). + workflow_run: + workflows: + - Custard CI # .github/workflows/custard-ci.yaml + types: + - in_progress + + # Run tests again as validation when a PR merge into main. + push: + branches: + - main + + # To do manual runs through the Actions UI. + workflow_dispatch: + inputs: + run-all: + description: Run all tests + type: boolean + default: false + paths: + description: Comma separated packages to test + type: string + ref: + description: Branch, tag, or commit SHA to run tests on + type: string + default: main + +jobs: + affected: + uses: GoogleCloudPlatform/cloud-samples-tools/.github/workflows/affected.yaml@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + permissions: + statuses: write + with: + head-sha: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + config-file: .github/config/nodejs-dev.jsonc + paths: ${{ (inputs.run-all && '.') || inputs.paths || '' }} + check-name: (dev) Custard CI / tests + create-check-if: ${{ !!github.event.workflow_run }} + + test: + if: needs.affected.outputs.paths != '[]' + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit + permissions: + id-token: write + statuses: write + strategy: + fail-fast: false + matrix: + path: ${{ needs.affected.outputs.paths }} + continue-on-error: true + env: + GOOGLE_SAMPLES_PROJECT: long-door-651 + SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + steps: + - name: Check queued + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/create-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + id: queued + with: + sha: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + name: (dev) Custard CI / ${{ github.job }} (${{ matrix.path }}) + job-name: ${{ github.job }} (${{ matrix.path }}) + if: ${{ !!github.event.workflow_run }} + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + - name: Authenticate + uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 + id: auth + with: + project_id: ${{ env.GOOGLE_SAMPLES_PROJECT }} + workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider + service_account: ${{ env.SERVICE_ACCOUNT }} + access_token_lifetime: 600s # 10 minutes + token_format: id_token + id_token_audience: https://action.test/ # service must have this custom audience + id_token_include_email: true + - name: Setup Custard + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/setup-custard@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + path: ${{ matrix.path }} + ci-setup: ${{ toJson(fromJson(needs.affected.outputs.ci-setups)[matrix.path]) }} + id-token: ${{ steps.auth.outputs.id_token }} + - name: Check in_progress + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + id: in_progress + with: + check: ${{ steps.queued.outputs.check }} + status: in_progress + - name: Run tests for ${{ matrix.path }} + run: | + timeout ${{ fromJson(needs.affected.outputs.ci-setups)[matrix.path].timeout-minutes }}m \ + make test dir=${{ matrix.path }} + env: + # TODO: remove this when the self-contained runner lands. + SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + - name: Check success + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + check: ${{ steps.in_progress.outputs.check }} + status: success + - name: Check failure + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + if: failure() + with: + check: ${{ steps.in_progress.outputs.check }} + status: failure + + done: + needs: [affected, test] + if: always() + runs-on: ubuntu-latest + permissions: + statuses: write + steps: + - name: Check success + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + check: ${{ needs.affected.outputs.check }} + status: success diff --git a/.github/workflows/custard-run.yaml b/.github/workflows/custard-run.yaml new file mode 100644 index 0000000000..c5fc5c94d2 --- /dev/null +++ b/.github/workflows/custard-run.yaml @@ -0,0 +1,186 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Custard run + +on: + # Run tests when a pull request is created or updated. + # This allows to run tests from forked repos (after reviewer's approval). + workflow_run: + workflows: + - Custard CI # .github/workflows/custard-ci.yaml + types: + - in_progress + + # Run tests again as validation when a PR merge into main. + push: + branches: + - main + + # To do manual runs through the Actions UI. + workflow_dispatch: + inputs: + run-all: + description: Run all tests + type: boolean + default: false + paths: + description: Comma separated packages to test + type: string + ref: + description: Branch, tag, or commit SHA to run tests on + type: string + default: main + + # For nightly tests. + # schedule: + # # https://crontab.guru/#0_12_*_*_0 + # - cron: 0 12 * * 0 # At 12:00 on Sunday + +jobs: + affected: + uses: GoogleCloudPlatform/cloud-samples-tools/.github/workflows/affected.yaml@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + permissions: + statuses: write + with: + head-sha: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + config-file: .github/config/nodejs.jsonc + paths: ${{ (inputs.run-all && '.') || inputs.paths || '' }} + check-name: Custard CI / tests + create-check-if: ${{ !!github.event.workflow_run }} + + lint: + needs: affected + runs-on: ubuntu-latest + permissions: + statuses: write + timeout-minutes: 5 + steps: + - name: Check in_progress + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/create-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + id: in_progress + with: + sha: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + status: in_progress + name: Custard CI / ${{ github.job }} + if: ${{ !!github.event.workflow_run }} + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + - name: Setup Node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 20 + - run: npm install + - name: npx gtx lint (${{ needs.affected.outputs.num-paths }} packages) + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/map-run@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + command: npx gts lint + paths: ${{ needs.affected.outputs.paths }} + - name: Check success + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + check: ${{ steps.in_progress.outputs.check }} + status: success + - name: Check failure + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + if: failure() + with: + check: ${{ steps.in_progress.outputs.check }} + status: failure + + test: + if: needs.affected.outputs.paths != '[]' + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit + permissions: + id-token: write + statuses: write + strategy: + fail-fast: false + matrix: + path: ${{ fromJson(needs.affected.outputs.paths) }} + continue-on-error: true + env: + GOOGLE_SAMPLES_PROJECT: long-door-651 + SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + steps: + - name: Check queued + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/create-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + id: queued + with: + sha: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + name: Custard CI / ${{ github.job }} (${{ matrix.path }}) + job-name: ${{ github.job }} (${{ matrix.path }}) + if: ${{ !!github.event.workflow_run }} + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.workflow_run.head_sha || inputs.ref || github.sha }} + - name: Authenticate + uses: google-github-actions/auth@ba79af03959ebeac9769e648f473a284504d9193 # v2.1.10 + id: auth + with: + project_id: ${{ env.GOOGLE_SAMPLES_PROJECT }} + workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider + service_account: ${{ env.SERVICE_ACCOUNT }} + access_token_lifetime: 600s # 10 minutes + token_format: id_token + id_token_audience: https://action.test/ # service must have this custom audience + id_token_include_email: true + - name: Setup Custard + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/setup-custard@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + path: ${{ matrix.path }} + ci-setup: ${{ toJson(fromJson(needs.affected.outputs.ci-setups)[matrix.path]) }} + id-token: ${{ steps.auth.outputs.id_token }} + - name: Check in_progress + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + id: in_progress + with: + check: ${{ steps.queued.outputs.check }} + status: in_progress + - name: Run tests for ${{ matrix.path }} + run: | + timeout ${{ fromJson(needs.affected.outputs.ci-setups)[matrix.path].timeout-minutes }}m \ + make test dir=${{ matrix.path }} + env: + # TODO: remove this when the self-contained runner lands. + SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com + - name: Check success + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + check: ${{ steps.in_progress.outputs.check }} + status: success + - name: Check failure + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + if: failure() + with: + check: ${{ steps.in_progress.outputs.check }} + status: failure + + done: + needs: [affected, lint, test] + if: always() + runs-on: ubuntu-latest + permissions: + statuses: write + steps: + - name: Check success + uses: GoogleCloudPlatform/cloud-samples-tools/actions/steps/update-check@9ee708234e240605d96e78f652c333ed6aa95a23 # v0.3.2 + with: + check: ${{ needs.affected.outputs.check }} + status: success diff --git a/.github/workflows/dataproc.yaml b/.github/workflows/dataproc.yaml new file mode 100644 index 0000000000..88e47368b2 --- /dev/null +++ b/.github/workflows/dataproc.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: dataproc +on: + push: + branches: + - main + paths: + - 'dataproc/**' + - '.github/workflows/dataproc.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'dataproc/**' + - '.github/workflows/dataproc.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'dataproc' + path: 'dataproc' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/datastore-functions.yaml b/.github/workflows/datastore-functions.yaml new file mode 100644 index 0000000000..3f0f8b8059 --- /dev/null +++ b/.github/workflows/datastore-functions.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: datastore-functions +on: + push: + branches: + - main + paths: + - 'datastore/functions/**' + - '.github/workflows/datastore-functions.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'datastore/functions/**' + - '.github/workflows/datastore-functions.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'datastore-functions' + path: 'datastore/functions' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/dialogflow-cx.yaml b/.github/workflows/dialogflow-cx.yaml new file mode 100644 index 0000000000..af31747cf9 --- /dev/null +++ b/.github/workflows/dialogflow-cx.yaml @@ -0,0 +1,100 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: dialogflow-cx +on: + push: + branches: + - main + paths: + - 'dialogflow-cx/**' + - '.github/workflows/dialogflow-cx.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'dialogflow-cx/**' + - '.github/workflows/dialogflow-cx.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce' # v2 + with: + secrets: |- + agent_id:nodejs-docs-samples-tests/nodejs-docs-samples-dialogflow-cx-agent-id + test_id:nodejs-docs-samples-tests/nodejs-docs-samples-dialogflow-cx-test-id + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=dialogflow-cx" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - name: Run Tests + run: make test dir=dialogflow-cx + env: + GOOGLE_SAMPLES_PROJECT: "long-door-651" + AGENT_ID: ${{ steps.secrets.outputs.agent_id }} + TEST_ID: ${{ steps.secrets.outputs.test_id }} + AGENT_PROJECT_ID: nodejs-docs-samples-tests + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: dialogflow-cx/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/dlp.yaml b/.github/workflows/dlp.yaml new file mode 100644 index 0000000000..8453ddf498 --- /dev/null +++ b/.github/workflows/dlp.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: dlp +on: + push: + branches: + - main + paths: + - 'dlp/**' + - '.github/workflows/dlp.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'dlp/**' + - '.github/workflows/dlp.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'dlp' + path: 'dlp' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/document-ai.yaml b/.github/workflows/document-ai.yaml new file mode 100644 index 0000000000..78ef3e388a --- /dev/null +++ b/.github/workflows/document-ai.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: document-ai +on: + push: + branches: + - main + paths: + - 'document-ai/**' + - '.github/workflows/document-ai.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'document-ai/**' + - '.github/workflows/document-ai.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'document-ai' + path: 'document-ai' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/flakybot.yaml b/.github/workflows/flakybot.yaml new file mode 100644 index 0000000000..a70cb87bba --- /dev/null +++ b/.github/workflows/flakybot.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: FlakyBot + +on: + workflow_call: + +env: + FLAKYBOT_VERSION: '1.2.0' + +jobs: + flakybot: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: DISABLED run FlakyBot + run: echo flakybot error reporting disabled \ No newline at end of file diff --git a/.github/workflows/functions-slack.yaml b/.github/workflows/functions-slack.yaml new file mode 100644 index 0000000000..7e32c75185 --- /dev/null +++ b/.github/workflows/functions-slack.yaml @@ -0,0 +1,99 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: functions-slack +on: + push: + branches: + - main + paths: + - 'functions/slack/**' + - '.github/workflows/functions-slack.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'functions/slack/**' + - '.github/workflows/functions-slack.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce' # v2 + with: + secrets: |- + slack_secret:nodejs-docs-samples-tests/nodejs-docs-samples-slack-secret + kg_api_key:nodejs-docs-samples-tests/nodejs-docs-samples-kg-api-key + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=functions-slack" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - name: Run Tests + run: make test dir=functions/slack + env: + GOOGLE_SAMPLES_PROJECT: "long-door-651" + SLACK_SECRET: ${{ steps.secrets.outputs.slack_secret }} + KG_API_KEY: ${{ steps.secrets.outputs.kg_api_key }} + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: functions/slack/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/healthcare-fhir.yaml b/.github/workflows/healthcare-fhir.yaml new file mode 100644 index 0000000000..0d1421b689 --- /dev/null +++ b/.github/workflows/healthcare-fhir.yaml @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: healthcare-fhir +on: + push: + branches: + - main + paths: + - 'healthcare/fhir/**' + - '.github/workflows/healthcare-fhir.yaml' + - '.github/workflows/test.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'healthcare/fhir/**' + - '.github/workflows/healthcare-fhir.yaml' + - '.github/workflows/test.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: 'healthcare-fhir' + path: 'healthcare/fhir' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/iam-deny.yaml b/.github/workflows/iam-deny.yaml new file mode 100644 index 0000000000..2e2e097bfb --- /dev/null +++ b/.github/workflows/iam-deny.yaml @@ -0,0 +1,97 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +name: iam-deny +on: + push: + branches: + - main + paths: + - 'iam/deny/**' + - '.github/workflows/iam-deny.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'iam/deny/**' + - '.github/workflows/iam-deny.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + defaults: + run: + working-directory: 'iam/deny' + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/949737848314/locations/global/workloadIdentityPools/iam-deny-test-pool/providers/iam-deny-test-provider' + service_account: 'kokoro-ca@isakovf-iam-deny-samples.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: "${{ runner.os }}-node- \n" + - name: install repo dependencies + run: npm install + working-directory: . + - name: install directory dependencies + run: npm install + - run: npm run build --if-present + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=iam-deny" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - run: npm test + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: iam/deny/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/remove-label.yaml b/.github/workflows/remove-label.yaml new file mode 100644 index 0000000000..c2913e8740 --- /dev/null +++ b/.github/workflows/remove-label.yaml @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Remove Label + +on: + workflow_call: + +permissions: {} + +jobs: + remove_label: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - run: echo "Label removal is currently skipped. Please remove the 'actions:force-run' label manually. See b/354216420" diff --git a/.github/workflows/storagetransfer.yaml b/.github/workflows/storagetransfer.yaml new file mode 100644 index 0000000000..0d73bdaae2 --- /dev/null +++ b/.github/workflows/storagetransfer.yaml @@ -0,0 +1,108 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: storagetransfer +on: + push: + branches: + - main + paths: + - 'storagetransfer/**' + - '.github/workflows/storagetransfer.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - 'storagetransfer/**' + - '.github/workflows/storagetransfer.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + defaults: + run: + working-directory: 'storagetransfer' + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce' # v2 + with: + secrets: |- + sts_aws_secret:nodejs-docs-samples-tests/nodejs-docs-samples-storagetransfer-aws + sts_azure_secret:nodejs-docs-samples-tests/nodejs-docs-samples-storagetransfer-azure + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: "${{ runner.os }}-node- \n" + - name: install repo dependencies + run: npm install + working-directory: . + - name: install directory dependencies + run: npm install + - run: npm run build --if-present + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=storagetransfer" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - run: npm test + env: + AWS_ACCESS_KEY_ID: ${{ fromJSON(steps.secrets.outputs.sts_aws_secret).AccessKeyId }} + AWS_SECRET_ACCESS_KEY: ${{ fromJSON(steps.secrets.outputs.sts_aws_secret).SecretAccessKey }} + AZURE_STORAGE_ACCOUNT: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).StorageAccount }} + AZURE_CONNECTION_STRING: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).ConnectionString }} + AZURE_SAS_TOKEN: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).SAS }} + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: storagetransfer/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..916708a517 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,74 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Test Sample + +on: + workflow_call: + inputs: + name: + required: true + type: string + path: + required: true + type: string +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa' # v2.1.3 + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: set testing env vars + run: echo "MOCHA_REPORTER_OUTPUT=${{github.run_id}}_sponge_log.xml" >> $GITHUB_ENV + - name: set testing env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME=${{ inputs.name }}" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - name: Run Tests + run: make test dir=${{ inputs.path }} + env: + GOOGLE_SAMPLES_PROJECT: "long-door-651" + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: test-results + path: ${{ inputs.path }}/${{ env.MOCHA_REPORTER_OUTPUT }} + retention-days: 1 diff --git a/.github/workflows/utils/ci-matrix.yaml.njk b/.github/workflows/utils/ci-matrix.yaml.njk new file mode 100644 index 0000000000..1322315ef4 --- /dev/null +++ b/.github/workflows/utils/ci-matrix.yaml.njk @@ -0,0 +1,56 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: {{name}} +on: + push: + branches: + - main + paths: + - '{{path}}/**' + - '.github/workflows/{{name}}.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - '{{path}}/**' + - '.github/workflows/{{name}}.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + strategy: + matrix: + path: + # TODO: Replace these examples below + - 'example/subfolder/1' + - 'example/subfolder/2' + uses: ./.github/workflows/test.yaml + with: + name: '{{name}}' + path: '${% raw %}{{ matrix.path }}{% endraw %}' + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/utils/ci-secrets.yaml.njk b/.github/workflows/utils/ci-secrets.yaml.njk new file mode 100644 index 0000000000..6589360010 --- /dev/null +++ b/.github/workflows/utils/ci-secrets.yaml.njk @@ -0,0 +1,101 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: {{ name }} +on: + push: + branches: + - main + paths: + - '{{ path }}/**' + - '.github/workflows/{{ name }}.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - '{{ path }}/**' + - '.github/workflows/{{ name }}.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: 'read' + id-token: 'write' + steps: + - uses: actions/checkout@v4.1.0 + with: + ref: ${% raw %}{{github.event.pull_request.head.sha}}{% endraw %} + - uses: 'google-github-actions/auth@v1.1.1' + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: 'google-github-actions/get-secretmanager-secrets@v1' + with: + secrets: |- + # TODO: Update secrets + secret_key_1:nodejs-docs-samples-tests/secret-1 + secret_key_2:nodejs-docs-samples-tests/secret-2 + - uses: actions/setup-node@v3.8.0 + with: + node-version: 16 + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${% raw %}{{ steps.npm-cache-dir.outputs.dir }}{% endraw %} + key: ${% raw %}{{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}{% endraw %} + restore-keys: | + ${% raw %}{{ runner.os }}-node-{% endraw %} + - name: set env vars for scheduled run + if: github.event.action == 'schedule' + run: | + echo "MOCHA_REPORTER_SUITENAME={{ name }}" >> $GITHUB_ENV + echo "MOCHA_REPORTER_OUTPUT=${% raw %}{{github.run_id}}{% endraw %}_sponge_log.xml" >> $GITHUB_ENV + echo "MOCHA_REPORTER=xunit" >> $GITHUB_ENV + - name: Run Tests + run: make test dir={{ path }} + env: + GOOGLE_SAMPLES_PROJECT: "long-door-651" + # TODO: Update environment variables + SECRET_1: ${% raw %}{{ steps.secrets.outputs.secret_key_1 }}{% endraw %} + SECRET_2: ${% raw %}{{ steps.secrets.outputs.secret_key_2 }}{% endraw %} + - name: upload test results for FlakyBot workflow + if: github.event.action == 'schedule' && always() + uses: actions/upload-artifact@v3 + env: + MOCHA_REPORTER_OUTPUT: "${{github.run_id}}_sponge_log.xml" + with: + name: test-results + path: {{ path }}/${% raw %}{{ env.MOCHA_REPORTER_OUTPUT }}{% endraw %} + retention-days: 1 + flakybot: + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/utils/ci.yaml.njk b/.github/workflows/utils/ci.yaml.njk new file mode 100644 index 0000000000..bee213a157 --- /dev/null +++ b/.github/workflows/utils/ci.yaml.njk @@ -0,0 +1,52 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: {{name}} +on: + push: + branches: + - main + paths: + - '{{path}}/**' + - '.github/workflows/{{name}}.yaml' + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + paths: + - '{{path}}/**' + - '.github/workflows/{{name}}.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + # Ref: https://github.com/google-github-actions/auth#usage + permissions: + contents: 'read' + id-token: 'write' + if: github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' + uses: ./.github/workflows/test.yaml + with: + name: '{{name}}' + path: '{{path}}' + flakybot: + # Ref: https://github.com/google-github-actions/auth#usage + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name == 'schedule' && always() # always() submits logs even if tests fail + uses: ./.github/workflows/flakybot.yaml + needs: [test] diff --git a/.github/workflows/utils/generate.js b/.github/workflows/utils/generate.js new file mode 100755 index 0000000000..9903f2038f --- /dev/null +++ b/.github/workflows/utils/generate.js @@ -0,0 +1,73 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const nunjucks = require('nunjucks'); +const {program, Option} = require('commander'); +const fs = require('fs').promises; + +// Three templates: +// standard: most samples use this workflow +// matrix: uses GitHub Actions matrix feature to run different permutations of the same workflow +// secrets: samples that use secrets to populate environment variables +const templates = { + standard: 'utils/ci.yaml.njk', + matrix: 'utils/ci-matrix.yaml.njk', + secrets: 'utils/ci-secrets.yaml.njk', +}; +const workflow_files = { + standard: './workflows.json', + matrix: './workflows-matrix.json', + secrets: './workflows-secrets.json', +}; +const workflows_dir = '.github/workflows'; + +async function main() { + nunjucks.configure(workflows_dir, {autoescape: true}); + + program + .addOption( + new Option('-t, --type