diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 4f4a48d..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,309 +0,0 @@ -# name of the workflow. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#name -name: CI -# running on push to main and develop branches or on pull reuqests or on manual trigger -on: - # manual trigger - workflow_dispatch: - inputs: - ssh_debug_enabled: - type: boolean - description: 'Run the build/test with ssh debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false - debug_deployment: - type: boolean - description: 'Run the pipeline with debug deployment enabled' - required: false - default: false - - # runnnig on push to main and develop branches - push: - branches: - - main - - develop - paths-ignore: - - '**/README.md' - - '.devcontainer/**' - # running on pull requests to main and develop branches - pull_request: - branches: - - main - - develop - paths-ignore: - - '**/README.md' - -# defining global environment variables for all jobs -env: - # define runner indexes for tests splitting and parallel execution - total-runners: 5 - # defining GitHub registry for docker images - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - -# build job definition with 7 steps: -# 1. Checkout repository -# 2. Cache Maven packages -# 3. Running Liquibase Quality Checks link - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html -# 4. Initialize CodeQL -# 5. Build with Maven -# 6. Perform CodeQL Analysis link - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning -# 7. Upload artifact -jobs: - build: - # build job will run on ubuntu-latest github-hosted runner - runs-on: ubuntu-latest - # defining permissions for the job - read contents, write packages, write id-token. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idpermissions - # security-events permission is required for CodeQL analysis - # enforcing policy for the job - only users with write access to the repository can trigger the job. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idenforce_admins - permissions: - actions: read - contents: read - packages: write - id-token: write - security-events: write - # defining strategy for the job - matrix strategy for codeQL analysis - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - # defining steps for the job as explained above - steps: - - name: Checkout repository - uses: actions/checkout@v3 # cache maven packages step - caching maven packages to speed up the build process. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows - - name: Cache Maven packages - uses: actions/cache@v3 # defining cache key and restore keys for the cache step. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key - with: - path: ~/.m2/repository # path to the directory where maven packages are stored - /root/.m2 in the container - key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-build- - # running liquibase quality checks step to ensure that the database changelogs are well-formed and follow best practices. - # Link to the documentation - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html - # Using the Liquibase Maven plugin, you can run Liquibase Quality Checks on your changelogs. - # The plugin is available in the Maven Central repository. Link to Maven Central - https://search.maven.org/artifact/org.liquibase/liquibase-maven-plugin - - name: Running Liquibase Quality Checks to ensure that the database changelogs are well-formed and follow best practices - run: | - mvn process-resources liquibase:checks.run - # runnning code scanning with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning - # first step is to initialize CodeQL - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} # defining the language for the CodeQL analysis - # debug: true # uncomment this line to enable debugging for CodeQL analysis step - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # autobuild with codeql - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - # performing Code Quality Analysis with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" # defining the language for the CodeQL analysis - - uses: actions/upload-artifact@v3 # uploading the artifact to the GitHub Artifacts. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts - with: - name: jar-artifact # naming the artifact jar file/s path - path: target/ # actual relative path to the artifact in the container - target/ - - unit-parallel-tests: - # unit-parallel-tests job will run on ubuntu-latest github-hosted runner - name: UNIT-PARALLEL-TESTS - runs-on: ubuntu-latest - needs: # needs build job and runner-indexes job to be completed before running the unit-parallel-tests job - - build - - runner-indexes - container: - image: ghcr.io/tsviz/tsvi-spring-test:v1.0.0 # using the base maven:3.8.7-openjdk-18 image + jq + curl + nodejs 18 - services: - # postgres service container - postgres: # service name - postgres. This name is used to access the service container from the job container as the host name. - image: postgres # running the job in a container - postgres link to the docker hub - https://hub.docker.com/_/postgres - env: - POSTGRES_PASSWORD: postgres # setting the password for the postgres database - # exposing the port 5432 of the postgres service container to the host machine - ports: - - 5432:5432 - # redis service container for caching session data - redis: # service name - redis. This name is used to access the service container from the job container as the host name. - image: redis # running the job in a container - redis link to the docker hub - https://hub.docker.com/_/redis - # exposing the port 6379 of the redis service container to the host machine - ports: - - 6379:6379 - # defining the job permissions - permissions: - contents: read # read access to the repository contents - packages: write # write access to the repository packages - id-token: write # write access to the repository id token - strategy: # defining the job to run in parallel with the matrix strategy and runner-indexes job output - fail-fast: true # cancels all in-progress jobs if any matrix job fails link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast - matrix: # defining the matrix strategy to run the job in parallel using x number of github-hosted runners defined in the env total_runners above - runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} # using the runner-indexes job output to define the matrix strategy - steps: - - name: Checkout repository # checkout the repository - uses: actions/checkout@v3.0.2 - # caching the maven packages to speed up the build process. - # Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows - - name: Cache Maven packages - uses: actions/cache@v3 # using the actions/cache@v3 action to cache the maven packages - with: - path: /root/.m2 # path to cache - key: ${{ runner.os }}-junit-${{ hashFiles('**/pom.xml') }} # key for restoring and saving the cache - restore-keys: ${{ runner.os }}-junit- # key for restoring the cache if no exact match is found - # In this step, we are downloading the latest artifact from the build job and storing it in the container - - run: | - # Download the latest tests results artifact number from the GitHub API using jq to parse the JSON response - curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{ github.repository }}/actions/artifacts | jq -r '.artifacts | sort_by(.created_at) | .[] | select(.name == "Test Results") | .id' > artifacts_list.txt - - LATEST_ARTIFACT_NUMBER=$(cut -d: -f 2 artifacts_list.txt | sort -n | tail -n 1) - - curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -L -o my_artifact.zip \ - https://api.github.com/repos/${{ github.repository }}/actions/artifacts/"${LATEST_ARTIFACT_NUMBER}"/zip - - mkdir test_results - unzip my_artifact.zip -d test_results 2> /dev/null || true - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.ssh_debug_enabled }} - - # split-tests action - splits the tests into x number of groups - # based on the total number of github-hosted runners and junit previous test results by time and line count. - # Link to the action - https://github.com/marketplace/actions/split-tests - - uses: chaosaffe/split-tests@v1-alpha.1 - id: split-tests - name: Split tests - with: - glob: src/test/**/**/*.java # glob pattern to match the test files - split-total: ${{ env.total-runners }} # total number of github-hosted runners - split-index: ${{ matrix.runner-index }} # current runner index - junit-path: test_results/*xml # path to the junit test results with wildcards to match all the files - line-count: true # split the tests based on the junit test results by line count - # run the tests in parallel looping through the test-suite output from the split-tests action - - run: 'echo "This runner will execute the following tests: ${{ steps.split-tests.outputs.test-suite }}"' - - run: | - LIST="${{ steps.split-tests.outputs.test-suite }}" - for file in $LIST - do - # sleep for 10 seconds to avoid timeout errors - sleep 10 - mvn -Dtest=$(basename $file | sed -e "s/.java/,/" | tr -d '\r\n') -e test -Dspring.datasource.url=${{ secrets.LIQUIBASE_COMMAND_URL }} -Dspring.datasource.username=${{ secrets.LIQUIBASE_COMMAND_USERNAME }} -Dspring.datasource.password=${{ secrets.LIQUIBASE_COMMAND_PASSWORD }} -Dspring.liquibase.change-log=classpath:db/changelog/changelog_version-3.3.xml -Dserver.port=8086 -Dspring.redis.host=redis -Dspring.redis.port=6379 -Dspring.redis.mode=standalone - done - - - uses: actions/upload-artifact@v3 # upload the test results as an artifact - with: - name: Test Results - path: ./target/surefire-reports # path to the test results - retention-days: 90 # retention period for the artifact in days. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts#about-workflow-artifact-retention - - publish-test-results: - needs: unit-parallel-tests - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Download test results - uses: actions/download-artifact@v2 - with: - name: Test Results - path: test_results - - - name: Publish Test Results - uses: dorny/test-reporter@v1.8.0 - if: success() || failure() - with: - reporter: java-junit - name: JUnit Test Results - path: test_results/*.xml - - build-and-publish-docker-image: # job to build the docker image and publish it to the GitHub Container Registry - runs-on: ubuntu-latest # using the latest ubuntu runner - outputs: - image_tag: ghcr.io/${{ github.repository }}:${{ github.run_number }} # output the image tag to be used in the build-and-publish-docker-image job - needs: [build, unit-parallel-tests] # this job needs build and unit-parallel-tests jobs as a requirement to run - if: github.ref == 'refs/heads/main' # run this job only when the branch is main branch and not on pull requests or other branches - https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context - # permissions for write acces to the packages and id-token and push access to the repository to create the container registry token - permissions: - packages: write - id-token: write - contents: write - - # steps to run the unit-parallel-tests job are as follows: - # 1. checkout the repository - # 2. download the jar artifact from the build job - # 3. use the docker layer caching action to speed up the docker image build process - # 4. build the docker image - # 5. log in to the GitHub Container Registry - # 6. push the docker image to the GitHub Container Registry - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - uses: actions/download-artifact@v1 - with: - name: jar-artifact - path: target/ - - # build the docker image using the Dockerfile in the root of the repository - # and tag it with the current run number from the github action workflow run - - name: Log in to the GH Container Registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # using the docker login action from the github marketplace - github.com/marketplace/actions/docker-login - with: - registry: ${{ env.REGISTRY }} # using the registry environment variable - username: ${{ github.actor }} # using the github.actor context - password: ${{ secrets.GITHUB_TOKEN }} # using the GITHUB_TOKEN secret - - - name: Build and push Docker image - id: build_image - uses: docker/build-push-action@v4 # using the docker build and push action from the github marketplace - github.com/marketplace/actions/build-and-push-docker-images - with: - context: . # using the current directory as the context - push: true # push the docker image to the registry - tags: | - ghcr.io/${{ github.repository }}:${{ github.run_number }} - ghcr.io/${{ github.repository }}:latest - cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:latest # use the docker layer caching to speed up the docker image build process - cache-to: type=inline - - runner-indexes: # job to generate the runner indexes for the unit-parallel-tests job - runs-on: ubuntu-latest - name: Generate runner indexes - outputs: - json: ${{ steps.generate-index-list.outputs.json }} # output the json with the runner indexes - steps: - - id: generate-index-list # generate the runner indexes and save them to the json file - run: | - MAX_INDEX=$((${{ env.total-runners }}-1)) # calculate the max index - INDEX_LIST=$(seq 0 ${MAX_INDEX}) # generate the list of indexes - INDEX_JSON=$(jq --null-input --compact-output '. |= [inputs]' <<< ${INDEX_LIST}) # convert the list to the json - echo "json=${INDEX_JSON}" >> $GITHUB_OUTPUT # save the json to the GITHUB_OUTPUT environment variable - - deploy: - needs: [build-and-publish-docker-image] # this job needs build-and-publish-docker-image job as a requirement to run - uses: ./.github/workflows/cd.yml - with: - # with tag from the build-and-publish-docker-image job in the output_tags step - image_tag: "${{ needs.build-and-publish-docker-image.outputs.image_tag }}" - debug: "${{ github.event.inputs.debug_deployment }}" - secrets: inherit \ No newline at end of file