From 0f0663266c3d64a15dfefde82b607041de97369d Mon Sep 17 00:00:00 2001 From: Maksim Khimchenko Date: Wed, 25 Jun 2025 12:47:53 +0300 Subject: [PATCH 01/27] fix ssh key exit Signed-off-by: Maksim Khimchenko (cherry picked from commit a2e53fd926279caa3b05ddd4ae0eb859104ec8f0) --- templates/Setup.gitlab-ci.yml | 65 ++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 765277b..f926886 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -60,42 +60,43 @@ before_script: # Add ssh keys - | - [[ -z "${SOURCE_REPO_SSH_KEY}" || -z "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" ]] && exit 0 + if [[ -n "${SOURCE_REPO_SSH_KEY}" || -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" ]]; then - eval $(ssh-agent) - trap "kill -3 ${SSH_AGENT_PID}" ERR EXIT HUP INT QUIT TERM - export SSH_KNOWN_HOSTS=~/.ssh/known_hosts - mkdir -p ~/.ssh - touch ~/.ssh/known_hosts + eval $(ssh-agent) + trap "kill -3 ${SSH_AGENT_PID}" ERR EXIT HUP INT QUIT TERM + export SSH_KNOWN_HOSTS=~/.ssh/known_hosts + mkdir -p ~/.ssh + touch ~/.ssh/known_hosts - if [[ -n "${SOURCE_REPO_SSH_KEY}" ]]; then - ssh-add - <<< "${SOURCE_REPO_SSH_KEY}" - if [[ -n "${SOURCE_REPO}" ]]; then - HOST=$(grep -oP '(?<=@)[^/:]+' <<< ${SOURCE_REPO}) - HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) - while IFS= read -r KEY_LINE; do - CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") - if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then - echo "$KEY_LINE" >> ~/.ssh/known_hosts - fi - done <<< "$HOST_KEYS" + if [[ -n "${SOURCE_REPO_SSH_KEY}" ]]; then + ssh-add - <<< "${SOURCE_REPO_SSH_KEY}" + if [[ -n "${SOURCE_REPO}" ]]; then + HOST=$(grep -oP '(?<=@)[^/:]+' <<< ${SOURCE_REPO}) + HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) + while IFS= read -r KEY_LINE; do + CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") + if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then + echo "$KEY_LINE" >> ~/.ssh/known_hosts + fi + done <<< "$HOST_KEYS" + fi fi - fi - if [[ -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" ]]; then - ssh-add - <<< "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" - if [[ -n "${SVACE_ANALYZE_HOST}" ]]; then - echo "Adding svace ssh key (ignoring errors)." - set +e - HOST=${SVACE_ANALYZE_HOST} - HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) - while IFS= read -r KEY_LINE; do - CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") - if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then - echo "$KEY_LINE" >> ~/.ssh/known_hosts - fi - done <<< "$HOST_KEYS" - set -e + if [[ -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" ]]; then + ssh-add - <<< "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" + if [[ -n "${SVACE_ANALYZE_HOST}" ]]; then + echo "Adding svace ssh key (ignoring errors)." + set +e + HOST=${SVACE_ANALYZE_HOST} + HOST_KEYS=$(ssh-keyscan -H "$HOST" 2>/dev/null) + while IFS= read -r KEY_LINE; do + CONSTANT_PART=$(awk '{print $2, $3}' <<< "$KEY_LINE") + if ! grep -q "$CONSTANT_PART" ~/.ssh/known_hosts; then + echo "$KEY_LINE" >> ~/.ssh/known_hosts + fi + done <<< "$HOST_KEYS" + set -e + fi fi fi From d429e72cd819a01c21cef615e40cc6858368b7d1 Mon Sep 17 00:00:00 2001 From: Nikolay Mordvintsev Date: Fri, 27 Jun 2025 10:32:51 +0300 Subject: [PATCH 02/27] fix registry auth Signed-off-by: Nikolay Mordvintsev --- templates/CVE_Scan.gitlab-ci.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index c9f3c24..5e15068 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -35,10 +35,25 @@ echo echo "=======================================================" echo - # Login to registries + # Preparing DOCKER_CONFIG and login to registries - | - echo ${PROD_REGISTRY_PASSWORD} | docker login --username="${PROD_REGISTRY_USER}" --password-stdin ${PROD_REGISTRY} - echo ${DEV_REGISTRY_PASSWORD} | docker login --username="${DEV_REGISTRY_USER}" --password-stdin ${DEV_REGISTRY} + echo "Preparing DOCKER_CONFIG and login to registries" + mkdir -p "${workdir}/docker" + cat > "${workdir}/docker/config.json" << EOL + { + "auths": { + "${PROD_REGISTRY}": { + "auth": "$(echo -n "${PROD_REGISTRY_USER}:${PROD_REGISTRY_PASSWORD}" | base64)" + }, + "${DEV_REGISTRY}": { + "auth": "$(echo -n "${DEV_REGISTRY_USER}:${DEV_REGISTRY_PASSWORD}" | base64)" + } + } + } + EOL + export DOCKER_CONFIG="${workdir}/docker" + echo docker login "${PROD_REGISTRY}" + echo docker login "${DEV_REGISTRY}" echo echo "=======================================================" echo From 2c6049e973317ba97361507eb244128cbcc9bb3c Mon Sep 17 00:00:00 2001 From: Vladimir Portnov Date: Mon, 30 Jun 2025 22:50:57 +0800 Subject: [PATCH 03/27] remove magic json (#25) Signed-off-by: Vladimir Portnov --- templates/CVE_Scan.gitlab-ci.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index 5e15068..f0605ce 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -39,21 +39,9 @@ - | echo "Preparing DOCKER_CONFIG and login to registries" mkdir -p "${workdir}/docker" - cat > "${workdir}/docker/config.json" << EOL - { - "auths": { - "${PROD_REGISTRY}": { - "auth": "$(echo -n "${PROD_REGISTRY_USER}:${PROD_REGISTRY_PASSWORD}" | base64)" - }, - "${DEV_REGISTRY}": { - "auth": "$(echo -n "${DEV_REGISTRY_USER}:${DEV_REGISTRY_PASSWORD}" | base64)" - } - } - } - EOL export DOCKER_CONFIG="${workdir}/docker" - echo docker login "${PROD_REGISTRY}" - echo docker login "${DEV_REGISTRY}" + echo ${PROD_REGISTRY_PASSWORD} | docker login --username="${PROD_REGISTRY_USER}" --password-stdin ${PROD_REGISTRY} + echo ${DEV_REGISTRY_PASSWORD} | docker login --username="${DEV_REGISTRY_USER}" --password-stdin ${DEV_REGISTRY} echo echo "=======================================================" echo From 2cdb21d42c0222e9b40f6957a1af21e269ff1a56 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 14 Jul 2025 10:04:39 +0300 Subject: [PATCH 04/27] add prod vals Signed-off-by: Pavel Okhlopkov --- templates/Setup.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index f926886..237cdc7 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -5,12 +5,11 @@ # $DEV_MODULES_REGISTRY - dev registry path # $DEV_MODULES_REGISTRY_LOGIN - login to dev registry # $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry +# $PROD_MODULES_REGISTRY - dev registry path +# $PROD_MODULES_REGISTRY_LOGIN - login to dev registry +# $PROD_MODULES_REGISTRY_PASSWORD - password to dev registry # $SOURCE_REPO - Source repository address for the module # $SOURCE_REPO_SSH_KEY - SSH private key for the source repository -# $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry -# $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry -# $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry -# $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry # SVACE_ANALYZE_HOST - hostname of the svace analyze vm # SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm # SVACE_ANALYZE_SSH_PRIVATE_KEY- svace analyze server ssh private key From f8c804aff81cca44d844539e71e1e496bd4f52c6 Mon Sep 17 00:00:00 2001 From: Pavel Okhlopkov Date: Mon, 14 Jul 2025 10:07:23 +0300 Subject: [PATCH 05/27] fix comments Signed-off-by: Pavel Okhlopkov --- templates/Setup.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 237cdc7..21a9387 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -5,9 +5,9 @@ # $DEV_MODULES_REGISTRY - dev registry path # $DEV_MODULES_REGISTRY_LOGIN - login to dev registry # $DEV_MODULES_REGISTRY_PASSWORD - password to dev registry -# $PROD_MODULES_REGISTRY - dev registry path -# $PROD_MODULES_REGISTRY_LOGIN - login to dev registry -# $PROD_MODULES_REGISTRY_PASSWORD - password to dev registry +# $PROD_MODULES_REGISTRY - prod registry path +# $PROD_MODULES_REGISTRY_LOGIN - login to prod registry +# $PROD_MODULES_REGISTRY_PASSWORD - password to prod registry # $SOURCE_REPO - Source repository address for the module # $SOURCE_REPO_SSH_KEY - SSH private key for the source repository # SVACE_ANALYZE_HOST - hostname of the svace analyze vm From 116d67571631fafb6fffdc6dcee43e9792567961 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Mon, 14 Jul 2025 11:31:27 +0300 Subject: [PATCH 06/27] edit tag for PRs Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 21a9387..d6c4117 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -22,6 +22,11 @@ variables: WERF_VERSION: "2 stable" before_script: + # setup tag + - | + if [[ ${CI_COMMIT_REF_NAME} != "main" && $CI_PIPELINE_SOURCE == "merge_request_event" ]]; then + export MODULES_MODULE_TAG=pr${CI_MERGE_REQUEST_IID} + fi # Setup trdl - | trdl_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) From f7fe7ad352e809dd6048b3e07f539e12676bb8b5 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Mon, 14 Jul 2025 11:42:26 +0300 Subject: [PATCH 07/27] MODULES_MODULE_TAG fix Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index d6c4117..dd9f047 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -22,11 +22,12 @@ variables: WERF_VERSION: "2 stable" before_script: - # setup tag + # Setup tag - | - if [[ ${CI_COMMIT_REF_NAME} != "main" && $CI_PIPELINE_SOURCE == "merge_request_event" ]]; then - export MODULES_MODULE_TAG=pr${CI_MERGE_REQUEST_IID} + if [[ ${CI_COMMIT_REF_NAME} != "main" ]]; then + export MODULES_MODULE_TAG=$(echo "${CI_COMMIT_REF_NAME}"|sed -e 's/\//-/g') fi + echo "MODULES_MODULE_TAG: ${MODULES_MODULE_TAG}" # Setup trdl - | trdl_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) @@ -48,6 +49,9 @@ before_script: if [[ "x${MODULES_REGISTRY_PASSWORD}" == "x" ]]; then MODULES_REGISTRY_PASSWORD="${CI_REGISTRY_PASSWORD}" fi + + echo "MODULES_REGISTRY_LOGIN: ${MODULES_REGISTRY_LOGIN}" + echo "MODULES_REGISTRY: ${MODULES_REGISTRY}" werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} if [[ -n "${DEV_MODULES_REGISTRY_LOGIN}" && -n "${DEV_MODULES_REGISTRY_PASSWORD}" && -n "${DEV_MODULES_REGISTRY}" ]]; then From 49f5191619c188daaa7159d13ab004bebf24e235 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Mon, 14 Jul 2025 19:48:59 +0300 Subject: [PATCH 08/27] add debug logs Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index dd9f047..74d2204 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -17,19 +17,21 @@ variables: MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" MODULES_MODULE_TAG: ${CI_COMMIT_REF_NAME} - BASE_IMAGES_VERSION: v0.5.7 + BASE_IMAGES_VERSION: v0.5.8 WERF_REPO: ${MODULES_MODULE_SOURCE}/${MODULES_MODULE_NAME} WERF_VERSION: "2 stable" before_script: # Setup tag - | + echo "=== Setup tag ===" if [[ ${CI_COMMIT_REF_NAME} != "main" ]]; then export MODULES_MODULE_TAG=$(echo "${CI_COMMIT_REF_NAME}"|sed -e 's/\//-/g') fi echo "MODULES_MODULE_TAG: ${MODULES_MODULE_TAG}" # Setup trdl - | + echo "=== Setup trdl ===" trdl_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_version/linux-amd64/bin/trdl" install -D trdl ~/bin/trdl @@ -38,6 +40,7 @@ before_script: # Setup werf - | + echo "=== Setup werf ===" trdl add werf https://tuf.werf.io 1 b7ff6bcbe598e072a86d595a3621924c8612c7e6dc6a82e919abe89707d7e3f468e616b5635630680dd1e98fc362ae5051728406700e6274c5ed1ad92bea52a2 source $(trdl use werf ${WERF_VERSION:-1.2 stable}) source $(werf ci-env gitlab --as-file) @@ -50,24 +53,33 @@ before_script: MODULES_REGISTRY_PASSWORD="${CI_REGISTRY_PASSWORD}" fi + - | + echo "=== Werf login ===" echo "MODULES_REGISTRY_LOGIN: ${MODULES_REGISTRY_LOGIN}" echo "MODULES_REGISTRY: ${MODULES_REGISTRY}" werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} + - | + echo "=== Werf dev login ===" if [[ -n "${DEV_MODULES_REGISTRY_LOGIN}" && -n "${DEV_MODULES_REGISTRY_PASSWORD}" && -n "${DEV_MODULES_REGISTRY}" ]]; then werf cr login -u ${DEV_MODULES_REGISTRY_LOGIN} -p ${DEV_MODULES_REGISTRY_PASSWORD} ${DEV_MODULES_REGISTRY} fi + # Setup dmt - | + echo "=== Werf dmt ===" trdl add dmt https://trrr.flant.dev/trdl-dmt/ 3 e77d785600a8c8612b84b93a5a2e4c48188d68f7478356d0708213e928bf67b024ed412e702dc32930da5c5bfc9b1c44be3ee7a292f923327815c91c6c3c3833 source $(trdl use dmt 0 stable) # Download base images yaml file - - env | grep BASE_IMAGES_VERSION - - curl --fail -sSLO https://fox.flant.com/api/v4/projects/deckhouse%2Fbase-images/packages/generic/base_images/${BASE_IMAGES_VERSION}/base_images.yml + - | + echo "=== Download base images yaml file ===" + echo "BASE_IMAGES_VERSION: ${BASE_IMAGES_VERSION}" + curl --fail -sSLO https://fox.flant.com/api/v4/projects/deckhouse%2Fbase-images/packages/generic/base_images/${BASE_IMAGES_VERSION}/base_images.yml # Add ssh keys - | + echo "=== Add ssh keys ===" if [[ -n "${SOURCE_REPO_SSH_KEY}" || -n "${SVACE_ANALYZE_SSH_PRIVATE_KEY}" ]]; then eval $(ssh-agent) From f84e6e2c994a8f4174ed4596df260e6b2c210468 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Mon, 14 Jul 2025 19:55:07 +0300 Subject: [PATCH 09/27] update werf login Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 74d2204..118efb9 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -52,16 +52,28 @@ before_script: if [[ "x${MODULES_REGISTRY_PASSWORD}" == "x" ]]; then MODULES_REGISTRY_PASSWORD="${CI_REGISTRY_PASSWORD}" fi - - - | + + - | # old logic echo "=== Werf login ===" - echo "MODULES_REGISTRY_LOGIN: ${MODULES_REGISTRY_LOGIN}" - echo "MODULES_REGISTRY: ${MODULES_REGISTRY}" - werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} + if [[ -n "${MODULES_REGISTRY_LOGIN}" && -n "${MODULES_REGISTRY_PASSWORD}" && -n "${MODULES_REGISTRY}" ]]; then + echo "MODULES_REGISTRY_LOGIN: ${MODULES_REGISTRY_LOGIN}" + echo "MODULES_REGISTRY: ${MODULES_REGISTRY}" + werf cr login -u ${MODULES_REGISTRY_LOGIN} -p ${MODULES_REGISTRY_PASSWORD} ${MODULES_REGISTRY} + fi + + - | + echo "=== Werf prod login ===" + if [[ -n "${PROD_MODULES_REGISTRY_LOGIN}" && -n "${PROD_MODULES_REGISTRY_PASSWORD}" && -n "${PROD_MODULES_REGISTRY}" ]]; then + echo "PROD_MODULES_REGISTRY_LOGIN: ${PROD_MODULES_REGISTRY_LOGIN}" + echo "PROD_MODULES_REGISTRY: ${PROD_MODULES_REGISTRY}" + werf cr login -u ${PROD_MODULES_REGISTRY_LOGIN} -p ${PROD_MODULES_REGISTRY_PASSWORD} ${PROD_MODULES_REGISTRY} + fi - | echo "=== Werf dev login ===" if [[ -n "${DEV_MODULES_REGISTRY_LOGIN}" && -n "${DEV_MODULES_REGISTRY_PASSWORD}" && -n "${DEV_MODULES_REGISTRY}" ]]; then + echo "DEV_MODULES_REGISTRY_LOGIN: ${DEV_MODULES_REGISTRY_LOGIN}" + echo "DEV_MODULES_REGISTRY: ${DEV_MODULES_REGISTRY}" werf cr login -u ${DEV_MODULES_REGISTRY_LOGIN} -p ${DEV_MODULES_REGISTRY_PASSWORD} ${DEV_MODULES_REGISTRY} fi From 4c4ade63113a69c64c272e31e1933d45020ed2e4 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Mon, 14 Jul 2025 23:02:17 +0300 Subject: [PATCH 10/27] Deny BE without ALLOW_BE=true Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 118efb9..1e33237 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -13,6 +13,8 @@ # SVACE_ANALYZE_HOST - hostname of the svace analyze vm # SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm # SVACE_ANALYZE_SSH_PRIVATE_KEY- svace analyze server ssh private key +# MODULE_EDITION (Optional) - one of [ce,fe,be,se,se-plus], only needed to check ALLOW_BE for the basic edition. +# ALLOW_BE (Optional) - if MODULE_EDITION=be, must be set to "true" for the build/deploy. variables: MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" @@ -22,6 +24,15 @@ variables: WERF_VERSION: "2 stable" before_script: + # Deny BE without ALLOW_BE=true + - | + echo "=== Check ALLOW_BE ===" + echo "MODULE_EDITION: ${MODULE_EDITION}" + echo "ALLOW_BE: ${ALLOW_BE}" + if [[ ${MODULE_EDITION} == "be" && ${ALLOW_BE} != "true" ]]; then + echo "BE edition don't allowed without ALLOW_BE=true" + exit 0 + fi # Setup tag - | echo "=== Setup tag ===" From a949af41406547f920e801bbfb0557d68591744d Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Wed, 16 Jul 2025 16:44:25 +0300 Subject: [PATCH 11/27] ALLOWED_EDITIONS suppport Signed-off-by: Smyslov Maxim --- templates/Build.gitlab-ci.yml | 1 + templates/Deploy.gitlab-ci.yml | 1 + templates/Setup.gitlab-ci.yml | 16 ++++++++-------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/templates/Build.gitlab-ci.yml b/templates/Build.gitlab-ci.yml index 0a83556..3182bed 100644 --- a/templates/Build.gitlab-ci.yml +++ b/templates/Build.gitlab-ci.yml @@ -27,6 +27,7 @@ variables: - if: $CI_COMMIT_TAG allow_failure: true +# before this script templates/Setup.gitlab-ci.yml will be executed .build: stage: build script: diff --git a/templates/Deploy.gitlab-ci.yml b/templates/Deploy.gitlab-ci.yml index 8ab84f6..ead7d2d 100644 --- a/templates/Deploy.gitlab-ci.yml +++ b/templates/Deploy.gitlab-ci.yml @@ -4,6 +4,7 @@ # $RELEASE_CHANNEL - lowercase release channel name, e.g., alpha, stable, early-access +# before this script templates/Setup.gitlab-ci.yml will be executed .deploy: stage: deploy script: diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 1e33237..7dbf68b 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -13,8 +13,8 @@ # SVACE_ANALYZE_HOST - hostname of the svace analyze vm # SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm # SVACE_ANALYZE_SSH_PRIVATE_KEY- svace analyze server ssh private key -# MODULE_EDITION (Optional) - one of [ce,fe,be,se,se-plus], only needed to check ALLOW_BE for the basic edition. -# ALLOW_BE (Optional) - if MODULE_EDITION=be, must be set to "true" for the build/deploy. +# MODULE_EDITION (Optional) - one of [ce,fe,be,ee,se,se-plus], only needed to allowance edition, if not set, allowance will be skipped. +# ALLOWED_EDITIONS - must set like "ce fe be ee se se-plus" (with spaces), used to check MODULE_EDITION is allowed. variables: MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" @@ -24,14 +24,14 @@ variables: WERF_VERSION: "2 stable" before_script: - # Deny BE without ALLOW_BE=true + # Deny edition without allowance - | - echo "=== Check ALLOW_BE ===" + echo "=== Check edition is allowed ===" + echo "ALLOWED_EDITIONS: ${ALLOWED_EDITIONS}" echo "MODULE_EDITION: ${MODULE_EDITION}" - echo "ALLOW_BE: ${ALLOW_BE}" - if [[ ${MODULE_EDITION} == "be" && ${ALLOW_BE} != "true" ]]; then - echo "BE edition don't allowed without ALLOW_BE=true" - exit 0 + if [[ -n $MODULE_EDITION && ! " ${ALLOWED_EDITIONS[@]} " =~ " ${MODULE_EDITION} " ]]; then + echo "Edition '${MODULE_EDITION}' is not allowed" + exit 77 fi # Setup tag - | From 062f78f23d1c413de89fb62fa4efdc058465bb53 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Wed, 16 Jul 2025 17:25:06 +0300 Subject: [PATCH 12/27] Setup completed log Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 7dbf68b..0cc0766 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -142,6 +142,7 @@ before_script: fi fi fi + - echo "=== Setup completed ===" stages: - build From 4209c17ac42efa295eda1264da03cd6fb7f36314 Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Wed, 16 Jul 2025 17:50:16 +0300 Subject: [PATCH 13/27] add -n $MODULE_EDITION comment Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 0cc0766..f358c75 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -29,6 +29,8 @@ before_script: echo "=== Check edition is allowed ===" echo "ALLOWED_EDITIONS: ${ALLOWED_EDITIONS}" echo "MODULE_EDITION: ${MODULE_EDITION}" + + # "-n $MODULE_EDITION" needed to skip validations for dev build/deploy if [[ -n $MODULE_EDITION && ! " ${ALLOWED_EDITIONS[@]} " =~ " ${MODULE_EDITION} " ]]; then echo "Edition '${MODULE_EDITION}' is not allowed" exit 77 From 356961bb77450458f2c2f4c5b02b95d104f3459d Mon Sep 17 00:00:00 2001 From: Smyslov Maxim Date: Wed, 16 Jul 2025 19:13:40 +0300 Subject: [PATCH 14/27] delete validation Signed-off-by: Smyslov Maxim --- templates/Setup.gitlab-ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index f358c75..a99567f 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -13,8 +13,6 @@ # SVACE_ANALYZE_HOST - hostname of the svace analyze vm # SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm # SVACE_ANALYZE_SSH_PRIVATE_KEY- svace analyze server ssh private key -# MODULE_EDITION (Optional) - one of [ce,fe,be,ee,se,se-plus], only needed to allowance edition, if not set, allowance will be skipped. -# ALLOWED_EDITIONS - must set like "ce fe be ee se se-plus" (with spaces), used to check MODULE_EDITION is allowed. variables: MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" @@ -24,17 +22,6 @@ variables: WERF_VERSION: "2 stable" before_script: - # Deny edition without allowance - - | - echo "=== Check edition is allowed ===" - echo "ALLOWED_EDITIONS: ${ALLOWED_EDITIONS}" - echo "MODULE_EDITION: ${MODULE_EDITION}" - - # "-n $MODULE_EDITION" needed to skip validations for dev build/deploy - if [[ -n $MODULE_EDITION && ! " ${ALLOWED_EDITIONS[@]} " =~ " ${MODULE_EDITION} " ]]; then - echo "Edition '${MODULE_EDITION}' is not allowed" - exit 77 - fi # Setup tag - | echo "=== Setup tag ===" From 3c8a13ba17a2e8590ae01a6fad91bbea9b3fa962 Mon Sep 17 00:00:00 2001 From: Alexandr Dyakonov <95070811+Suselz@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:05:05 +0500 Subject: [PATCH 15/27] [feature] Add downloading deckhouse lib-helm file (#27) Signed-off-by: Alexandr Dyakonov (cherry picked from commit 51ec3ab927a5a65a0088bfd86e9e33071eed1784) --- templates/Setup.gitlab-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index a99567f..8efc7bc 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -12,7 +12,8 @@ # $SOURCE_REPO_SSH_KEY - SSH private key for the source repository # SVACE_ANALYZE_HOST - hostname of the svace analyze vm # SVACE_ANALYZE_SSH_USER: - ssh user to connect with to svace analyze vm -# SVACE_ANALYZE_SSH_PRIVATE_KEY- svace analyze server ssh private key +# SVACE_ANALYZE_SSH_PRIVATE_KEY - svace analyze server ssh private key +# DECKHOUSE_LIB_HELM_VERSION - version of deckhouse lib-helm that will be downloaded variables: MODULES_MODULE_NAME: "${CI_PROJECT_NAME}" @@ -89,6 +90,13 @@ before_script: echo "BASE_IMAGES_VERSION: ${BASE_IMAGES_VERSION}" curl --fail -sSLO https://fox.flant.com/api/v4/projects/deckhouse%2Fbase-images/packages/generic/base_images/${BASE_IMAGES_VERSION}/base_images.yml + # Download deckhouse lib-helm archive + - | + if [[ -n "${DECKHOUSE_LIB_HELM_VERSION}" ]]; then + mkdir charts + curl --fail -sSLO https://github.com/deckhouse/lib-helm/releases/download/deckhouse_lib_helm-${DECKHOUSE_LIB_HELM_VERSION}/deckhouse_lib_helm-${DECKHOUSE_LIB_HELM_VERSION}.tgz --output-dir ./charts + fi + # Add ssh keys - | echo "=== Add ssh keys ===" From d5c82e999442fb28c55ce66011cc8817bf4553b6 Mon Sep 17 00:00:00 2001 From: Maksim Khimchenko <39365040+himax1991@users.noreply.github.com> Date: Wed, 30 Jul 2025 13:17:51 +0300 Subject: [PATCH 16/27] align import request to svacer 11 api (#26) Signed-off-by: Maksim Khimchenko (cherry picked from commit 3b0c0c0d11cc78c268cab12c561211fa3de5ac05) --- templates/Svace_Analayze.gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/templates/Svace_Analayze.gitlab-ci.yml b/templates/Svace_Analayze.gitlab-ci.yml index 747b94a..49b99c8 100644 --- a/templates/Svace_Analayze.gitlab-ci.yml +++ b/templates/Svace_Analayze.gitlab-ci.yml @@ -217,9 +217,11 @@ variables: --form project=\"${project_name}\" \ --form branch=\"${branch_name}\" \ --form file=@\"${archive_name}\" \ - --form options=\"--project-group ${PROJECT_GROUP}\" \ - --form options=\"--if-no-group ${IF_NO_GROUP}\" \ - --form options=\"--field COMMIT_HASH:${COMMIT_HASH}\"" + --form options='{\"values\":[ \ + {\"option\":\"project-group\",\"value\":\"${PROJECT_GROUP}\"}, \ + {\"option\":\"if-no-group\",\"value\":\"${IF_NO_GROUP}\"}, \ + {\"option\":\"field\",\"value\":\"CI_COMMIT_HASH:${CI_COMMIT_HASH}\"} \ + ]}'" info "Importing \"${project_name}\"..." response=$(send_request "${svacer_import}" $request_attempts) From c45ad72246072b087074b13f08e16420b234a2a8 Mon Sep 17 00:00:00 2001 From: Nikolay Mordvintsev Date: Thu, 21 Aug 2025 22:37:06 +0300 Subject: [PATCH 17/27] license scan Signed-off-by: Nikolay Mordvintsev --- templates/CVE_Scan.gitlab-ci.yml | 99 ++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index f0605ce..bf0009b 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -97,6 +97,49 @@ fi echo "CVE Scan will be applied to the following tags of ${MODULE_NAME} module:" echo "${module_tags[@]}" + + # Functions + trivy_scan() { + ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} --output ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + } + + send_report() { + echo "" + echo " Uploading trivy ${1} report for image \"${IMAGE_NAME}\" of \"${MODULE_NAME}\" module" + echo "" + curl -s -S -o /dev/null --fail-with-body -X POST \ + --retry 5 \ + --retry-delay 10 \ + --retry-all-errors \ + ${DD_URL}/api/v2/reimport-scan/ \ + -H "accept: application/json" \ + -H "Content-Type: multipart/form-data" \ + -H "Authorization: Token ${DD_TOKEN}" \ + -F "auto_create_context=True" \ + -F "minimum_severity=Info" \ + -F "active=true" \ + -F "verified=true" \ + -F "scan_type=Trivy Scan" \ + -F "close_old_findings=true" \ + -F "do_not_reactivate=false" \ + -F "push_to_jira=false" \ + -F "file=@${2}" \ + -F "product_type_name=External Modules" \ + -F "product_name=$MODULE_NAME" \ + -F "scan_date=${date_iso}" \ + -F "engagement_name=${1}" \ + -F "service=${MODULE_NAME} / ${IMAGE_NAME}" \ + -F "group_by=component_name+component_version" \ + -F "deduplication_on_engagement=false" \ + -F "tags=external_module,module:${MODULE_NAME},image:${IMAGE_NAME},branch:${module_tag},${dd_short_release_tag},${dd_full_release_tag},${dd_default_branch_tag}" \ + -F "test_title=[${MODULE_NAME}]: ${IMAGE_NAME}:${module_tag}" \ + -F "version=${dd_image_version}" \ + -F "build_id=${IMAGE_HASH}" \ + -F "commit_hash=${CI_COMMIT_SHA}" \ + -F "branch_tag=${module_tag}" \ + -F "apply_tags_to_findings=true" + } + # Scan in loop for provided list of tags for module_tag in ${module_tags[@]}; do dd_default_branch_tag="" @@ -154,50 +197,30 @@ if [ "${additional_image_detected}" == true ]; then if [ "${TRIVY_REPORTS_LOG_OUTPUT}" != "false" ]; then - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity ${severity} --ignorefile "${module_workdir}/.trivyignore" --format table --scanners vuln --quiet "${module_image}:${module_tag}" --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + # CVE Scan + trivy_scan "table" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}" + # License scan + trivy_scan "table" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}" fi - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity $severity --ignorefile "${module_workdir}/.trivyignore" --format json --scanners vuln --output "${module_reports}/d8_${MODULE_NAME}_${IMAGE_NAME}_report.json" --quiet "${module_image}:${module_tag}" --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + # CVE Scan + trivy_scan "json" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}" + # License scan + trivy_scan "json" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}" else if [ "${TRIVY_REPORTS_LOG_OUTPUT}" != "false" ]; then - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity ${severity} --ignorefile "${module_workdir}/.trivyignore" --format table --scanners vuln --quiet "${module_image}@${IMAGE_HASH}" --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + # CVE Scan + trivy_scan "table" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}" + # License scan + trivy_scan "table" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}" fi - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity ${severity} --ignorefile "${module_workdir}/.trivyignore" --format json --scanners vuln --output "${module_reports}/d8_${MODULE_NAME}_${IMAGE_NAME}_report.json" --quiet "${module_image}@${IMAGE_HASH}" --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + # CVE Scan + trivy_scan "json" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}" + # License scan + trivy_scan "json" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}" fi echo " Done" - echo "" - echo " Uploading trivy CVE report for image ${IMAGE_NAME} of ${MODULE_NAME} module" - echo "" - curl -s -S -o /dev/null --fail-with-body -X POST \ - --retry 5 \ - --retry-delay 10 \ - --retry-all-errors \ - ${DD_URL}/api/v2/reimport-scan/ \ - -H "accept: application/json" \ - -H "Content-Type: multipart/form-data" \ - -H "Authorization: Token ${DD_TOKEN}" \ - -F "auto_create_context=True" \ - -F "minimum_severity=Info" \ - -F "active=true" \ - -F "verified=true" \ - -F "scan_type=Trivy Scan" \ - -F "close_old_findings=true" \ - -F "do_not_reactivate=false" \ - -F "push_to_jira=false" \ - -F "file=@${module_reports}/d8_${MODULE_NAME}_${IMAGE_NAME}_report.json" \ - -F "product_type_name=Deckhouse images" \ - -F "product_name=$MODULE_NAME" \ - -F "scan_date=${date_iso}" \ - -F "engagement_name=CVE Test: ${MODULE_NAME} Images" \ - -F "service=${MODULE_NAME} / ${IMAGE_NAME}" \ - -F "group_by=component_name+component_version" \ - -F "deduplication_on_engagement=false" \ - -F "tags=deckhouse_image,module:${MODULE_NAME},image:${IMAGE_NAME},branch:${module_tag},${dd_short_release_tag},${dd_full_release_tag},${dd_default_branch_tag}" \ - -F "test_title=[${MODULE_NAME}]: ${IMAGE_NAME}:${module_tag}" \ - -F "version=${dd_image_version}" \ - -F "build_id=${IMAGE_HASH}" \ - -F "commit_hash=${CI_COMMIT_SHA}" \ - -F "branch_tag=${module_tag}" \ - -F "apply_tags_to_findings=true" + send_report "CVE" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" + send_report "License" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" done < <(jq -rc 'to_entries[]' <<< "${digests}") done rm -rf ${workdir} From 8ddf103f94b1e52ebb73b71c918a0c5dbf5a288d Mon Sep 17 00:00:00 2001 From: Nikolay Mordvintsev Date: Tue, 26 Aug 2025 17:38:24 +0300 Subject: [PATCH 18/27] fix cve output Signed-off-by: Nikolay Mordvintsev --- templates/CVE_Scan.gitlab-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index bf0009b..0f58899 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -100,7 +100,7 @@ # Functions trivy_scan() { - ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} --output ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote + ${workdir}/bin/trivy i --policy "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote } send_report() { @@ -198,25 +198,25 @@ if [ "${additional_image_detected}" == true ]; then if [ "${TRIVY_REPORTS_LOG_OUTPUT}" != "false" ]; then # CVE Scan - trivy_scan "table" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}" + trivy_scan "table" "--scanners vuln" "" "${module_image}:${module_tag}" # License scan - trivy_scan "table" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}" + trivy_scan "table" "--scanners license --license-full" "" "${module_image}:${module_tag}" fi # CVE Scan - trivy_scan "json" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}" + trivy_scan "json" "--scanners vuln" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}" # License scan - trivy_scan "json" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}" + trivy_scan "json" "--scanners license --license-full" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}" else if [ "${TRIVY_REPORTS_LOG_OUTPUT}" != "false" ]; then # CVE Scan - trivy_scan "table" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}" + trivy_scan "table" "--scanners vuln" "" "${module_image}@${IMAGE_HASH}" # License scan - trivy_scan "table" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}" + trivy_scan "table" "--scanners license --license-full" "" "${module_image}@${IMAGE_HASH}" fi # CVE Scan - trivy_scan "json" "--scanners vuln" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}" + trivy_scan "json" "--scanners vuln" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}" # License scan - trivy_scan "json" "--scanners license --license-full" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}" + trivy_scan "json" "--scanners license --license-full" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}" fi echo " Done" send_report "CVE" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" From dbd2e4893a797a057b45628baf539c589ea41c00 Mon Sep 17 00:00:00 2001 From: Maksim Khimchenko <39365040+himax1991@users.noreply.github.com> Date: Thu, 2 Oct 2025 12:19:17 +0300 Subject: [PATCH 19/27] Support multiple Svace versions (#42) Signed-off-by: Maksim Khimchenko (cherry picked from commit cf58880bcadb33798bd7bdd77fe0166db003cd74) Signed-off-by: Maksim Khimchenko --- templates/Svace_Analayze.gitlab-ci.yml | 50 +++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/templates/Svace_Analayze.gitlab-ci.yml b/templates/Svace_Analayze.gitlab-ci.yml index 49b99c8..1961b44 100644 --- a/templates/Svace_Analayze.gitlab-ci.yml +++ b/templates/Svace_Analayze.gitlab-ci.yml @@ -220,7 +220,8 @@ variables: --form options='{\"values\":[ \ {\"option\":\"project-group\",\"value\":\"${PROJECT_GROUP}\"}, \ {\"option\":\"if-no-group\",\"value\":\"${IF_NO_GROUP}\"}, \ - {\"option\":\"field\",\"value\":\"CI_COMMIT_HASH:${CI_COMMIT_HASH}\"} \ + {\"option\":\"field\",\"value\":\"CI_COMMIT_HASH:${COMMIT_HASH}\"}, \ + {\"option\":\"field\",\"value\":\"CI_COMMIT_REF_NAME:${COMMIT_REF_NAME}\"} \ ]}'" info "Importing \"${project_name}\"..." @@ -261,9 +262,43 @@ variables: fi } + define_import_params() { + local -n project="${1}" + local -n branch="${2}" + if [[ $(send "[[ -f ${proj}/.svace-dir/import-settings ]] && echo true || echo false") == true ]]; then + import_settings=$(send "cat /${proj}/.svace-dir/import-settings") + while IFS='=' read -r key val; do + case "$key" in + ProjectName) + project=$val + ;; + Branch) + branch=$val + ;; + *) + warning "Undefined import setting \"${key}=${val}\" will be ommited!" + ;; + esac + done <<< "${import_settings}" + fi + } + + get_svace_bin() { + proj="${1}" + svace_version=$(send "cat ${proj}/.svace-dir/svace-dir.version | awk 'FNR==3{print}'") + + svace_bin="/opt/svace-${svace_version}/bin/svace" + if [[ $(send "[[ -x ${svace_bin} ]] && echo true || echo false") == true ]]; then + echo "${svace_bin}" + else + echo "svace" + error "\"${svace_bin}\" is not executable on analyze server. Using default." + fi + } + echo "Searching for current build artifacts on server by path: /${SVACE_ANALYZE_DIR}/${COMMIT_HASH}" if [[ $(send "[[ -d /${SVACE_ANALYZE_DIR}/${COMMIT_HASH} ]] && echo true || echo false") == false ]]; then - echo "::warning file=$(realpath "$0")::Specified commit directory doesn't exists on analyze server." && exit 0 + warning "Specified commit directory doesn't exists on analyze server." && exit 0 fi projects=$(send "find /${SVACE_ANALYZE_DIR}/${COMMIT_HASH} \\( -type d -iname .svace-dir -o -iname *.tar.gz \\) -exec dirname {} \\;") @@ -275,10 +310,17 @@ variables: svacer_proj=${proj#/${SVACE_ANALYZE_DIR}/${COMMIT_HASH}/} build_item=${svacer_proj##*/} + import_project=${svacer_proj} + import_branch=${COMMIT_REF_NAME} + if [[ $(send "[[ -d ${proj}/.svace-dir ]] && echo true || echo false") == true ]]; then + define_import_params import_project import_branch + svace_bin=$(get_svace_bin "${proj}") + info "Using svace binary: $svace_bin" + info "Start analyzing project \"${svacer_proj}\" ..." - send "svace config --svace-dir ${proj} THREAD_NUMBER auto" - send "svace analyze --set-config SKIP_UNREACHABLE_PROCEDURE_ANALYSIS=${SKIP_UNREACHABLE_PROCEDURE_ANALYSIS} --quiet --svace-dir ${proj}" + send "${svace_bin} config --svace-dir ${proj} THREAD_NUMBER auto" + send "${svace_bin} analyze --set-config SKIP_UNREACHABLE_PROCEDURE_ANALYSIS=${SKIP_UNREACHABLE_PROCEDURE_ANALYSIS} --quiet --svace-dir ${proj}" success "Analysis completed successfully!" info "Start archiving project \"${svacer_proj}\" ..." From 6e97c5542d98949188f07330df4fa8cdf0dd901a Mon Sep 17 00:00:00 2001 From: Maksim Khimchenko <39365040+himax1991@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:09:39 +0300 Subject: [PATCH 20/27] Termination with error if Svace has nothing to analyze (#46) Signed-off-by: Maksim Khimchenko (cherry picked from commit 326a6f1194e95fb414f6f97230311083bf68c301) --- templates/Svace_Analayze.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/Svace_Analayze.gitlab-ci.yml b/templates/Svace_Analayze.gitlab-ci.yml index 1961b44..0ce0bf9 100644 --- a/templates/Svace_Analayze.gitlab-ci.yml +++ b/templates/Svace_Analayze.gitlab-ci.yml @@ -298,7 +298,7 @@ variables: echo "Searching for current build artifacts on server by path: /${SVACE_ANALYZE_DIR}/${COMMIT_HASH}" if [[ $(send "[[ -d /${SVACE_ANALYZE_DIR}/${COMMIT_HASH} ]] && echo true || echo false") == false ]]; then - warning "Specified commit directory doesn't exists on analyze server." && exit 0 + warning "Specified commit directory doesn't exists on analyze server." && exit 1 fi projects=$(send "find /${SVACE_ANALYZE_DIR}/${COMMIT_HASH} \\( -type d -iname .svace-dir -o -iname *.tar.gz \\) -exec dirname {} \\;") From 4f15c92936054a212457105f8459a7b368c39b6a Mon Sep 17 00:00:00 2001 From: Vladimir Portnov Date: Thu, 9 Oct 2025 21:04:05 +0800 Subject: [PATCH 21/27] Add trdl version check (#47) Signed-off-by: Vladimir Portnov --- templates/Setup.gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 8efc7bc..3626ef2 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -33,11 +33,14 @@ before_script: # Setup trdl - | echo "=== Setup trdl ===" - trdl_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) - curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_version/linux-amd64/bin/trdl" - install -D trdl ~/bin/trdl - rm trdl export PATH=$PATH:~/bin + trdl_target_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) + trdl_installed_version=$(trdl | tail -n 1 | sed -E 's/Version: v([0-9\.]+)/\1/') + if [[ "$trdl_installed_version" != "$trdl_target_version" ]]; then + curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_target_version/linux-amd64/bin/trdl" + install -D trdl ~/bin/trdl + rm trdl + fi # Setup werf - | From 7c6d84465b68327294c2dc7e6a09fed4dc3e4459 Mon Sep 17 00:00:00 2001 From: rtrofimenkov-ssdlc Date: Thu, 9 Oct 2025 18:43:12 +0500 Subject: [PATCH 22/27] fix: remove incorrect Content-Type header from CVE scan Signed-off-by: Roman Trofimenkov --- templates/CVE_Scan.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index 0f58899..defb622 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -113,7 +113,6 @@ --retry-all-errors \ ${DD_URL}/api/v2/reimport-scan/ \ -H "accept: application/json" \ - -H "Content-Type: multipart/form-data" \ -H "Authorization: Token ${DD_TOKEN}" \ -F "auto_create_context=True" \ -F "minimum_severity=Info" \ From 371126b4b2cb10a1d007b19d22cf44cfb194787e Mon Sep 17 00:00:00 2001 From: Roman Trofimenkov Date: Fri, 10 Oct 2025 14:50:39 +0500 Subject: [PATCH 23/27] fix: update do_not_reactivate flag in CVE scan configuration Signed-off-by: Roman Trofimenkov --- templates/CVE_Scan.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index defb622..fb5c3cb 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -120,7 +120,7 @@ -F "verified=true" \ -F "scan_type=Trivy Scan" \ -F "close_old_findings=true" \ - -F "do_not_reactivate=false" \ + -F "do_not_reactivate=true" \ -F "push_to_jira=false" \ -F "file=@${2}" \ -F "product_type_name=External Modules" \ From 5355bd8a2f0e970c555c27a7a8f42bc9ac117fee Mon Sep 17 00:00:00 2001 From: Nikolay Mordvintsev Date: Sat, 11 Oct 2025 12:59:58 +0300 Subject: [PATCH 24/27] do_not_reactivate=false Signed-off-by: Nikolay Mordvintsev --- templates/CVE_Scan.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/CVE_Scan.gitlab-ci.yml b/templates/CVE_Scan.gitlab-ci.yml index fb5c3cb..defb622 100644 --- a/templates/CVE_Scan.gitlab-ci.yml +++ b/templates/CVE_Scan.gitlab-ci.yml @@ -120,7 +120,7 @@ -F "verified=true" \ -F "scan_type=Trivy Scan" \ -F "close_old_findings=true" \ - -F "do_not_reactivate=true" \ + -F "do_not_reactivate=false" \ -F "push_to_jira=false" \ -F "file=@${2}" \ -F "product_type_name=External Modules" \ From 79c31f8a7df2633e93150bc7dd160d5563e5e097 Mon Sep 17 00:00:00 2001 From: rtrofimenkov-ssdlc Date: Fri, 17 Oct 2025 14:45:46 +0500 Subject: [PATCH 25/27] Add GitLab CI template for secret scanning with Gitleaks (#44) * gitleaks template Signed-off-by: Roman Trofimenkov * path fix, added stage Signed-off-by: Roman Trofimenkov * added docker runner tag Signed-off-by: Roman Trofimenkov * tags fix Signed-off-by: Roman Trofimenkov * pipeline refactor for shell executor Signed-off-by: Roman Trofimenkov * gitleaks pipe refactor Signed-off-by: Roman Trofimenkov * Add empty before_script to gitleaks CI template Signed-off-by: Roman Trofimenkov * PATH fix Signed-off-by: Roman Trofimenkov * Add gitleaks cleanup stage to CI template Signed-off-by: Roman Trofimenkov * Update gitleaks CI template to include optional dependencies and rules for cleanup stage Signed-off-by: Roman Trofimenkov * updated error parsind and printing to stdout Signed-off-by: Roman Trofimenkov * stdout fix Signed-off-by: Roman Trofimenkov * output fix Signed-off-by: Roman Trofimenkov * stdout fix Signed-off-by: Roman Trofimenkov * fixed cleanup stage Signed-off-by: Roman Trofimenkov * cleanup fix Signed-off-by: Roman Trofimenkov * cleanup stage fix Signed-off-by: Roman Trofimenkov * depth fix Signed-off-by: Roman Trofimenkov * deleted cleanup stage Signed-off-by: Roman Trofimenkov * report fix Signed-off-by: Roman Trofimenkov * Refactor cleanup process in GitLab CI configuration to use after_script for better clarity and organization. Signed-off-by: Roman Trofimenkov * Remove redundant stages declaration from gitleaks template Signed-off-by: Roman Trofimenkov * fix: add GitLab server host to gitleaks blob URLs Signed-off-by: Roman Trofimenkov --------- Signed-off-by: Roman Trofimenkov --- templates/gitleaks.gitlab-ci.yml | 131 +++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 templates/gitleaks.gitlab-ci.yml diff --git a/templates/gitleaks.gitlab-ci.yml b/templates/gitleaks.gitlab-ci.yml new file mode 100644 index 0000000..74fc069 --- /dev/null +++ b/templates/gitleaks.gitlab-ci.yml @@ -0,0 +1,131 @@ +variables: + GITLEAKS_VERSION: "v8.28.0" + +.gitleaks_scan: + stage: gitleaks + before_script: [] + script: + - | + set -euo pipefail + + # ========== Install Gitleaks ========== + echo "๐Ÿ“ฅ Installing Gitleaks $GITLEAKS_VERSION..." + file_ver="${GITLEAKS_VERSION#v}" + arch="$(uname -m)" + case "$arch" in + x86_64|amd64) pkg_arch="linux_x64" ;; + aarch64|arm64) pkg_arch="linux_arm64" ;; + *) echo "Unsupported arch: $arch"; exit 1 ;; + esac + + base="https://github.com/gitleaks/gitleaks/releases/download/$%7BGITLEAKS_VERSION%7D" + tgz="gitleaks_${file_ver}_${pkg_arch}.tar.gz" + curl -sSL "$base/$tgz" -o gitleaks.tgz + tar -xzf gitleaks.tgz gitleaks + chmod +x gitleaks + mkdir -p "$HOME/.local/bin" + mv gitleaks "$HOME/.local/bin/" + export PATH="$HOME/.local/bin:$PATH" + gitleaks version + + # ========== Check for config ========== + if [[ -f "gitleaks.toml" ]]; then + CONFIG_ARG="-c gitleaks.toml" + echo "โœ… Found config: gitleaks.toml" + else + CONFIG_ARG="" + echo "โš ๏ธ Config file not found. Proceeding with default rules." + fi + + # ========== Run scan ========== + GITLEAKS_EXIT=0 + if [[ "$SCAN_MODE" == "diff" ]]; then + echo "๐Ÿ•ต๏ธ Running in DIFF mode..." + git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=1 + BASE_SHA=$(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD || echo "") + if [[ -z "$BASE_SHA" ]]; then + echo "โŒ BASE_SHA not found. Aborting." + exit 1 + fi + echo "โ–ถ Scanning diff: $BASE_SHA...HEAD" + gitleaks detect --no-banner --report-format json --report-path gitleaks.json $CONFIG_ARG --source . --log-opts "$BASE_SHA...HEAD" || GITLEAKS_EXIT=$? + elif [[ "$SCAN_MODE" == "full" ]]; then + echo "๐Ÿ•ต๏ธ Running in FULL mode..." + gitleaks detect --no-banner --report-format json --report-path gitleaks.json $CONFIG_ARG --source . || GITLEAKS_EXIT=$? + else + echo "โŒ Unknown SCAN_MODE: $SCAN_MODE" + exit 1 + fi + + echo "๐Ÿ” Gitleaks exit code: $GITLEAKS_EXIT" + + # ========== Parse and print results ========== + echo "๐Ÿ“ค Parsing gitleaks.json for CI log output..." + echo "DEBUG: Checking gitleaks.json file..." + ls -lh gitleaks.json || echo "โš ๏ธ gitleaks.json not found!" + + if [[ -s gitleaks.json ]]; then + COUNT=$(jq length gitleaks.json) + echo "โŒ Leaks found: $COUNT" + echo "" + echo "DEBUG: Attempting to parse and display leaks..." + + jq -r ' + def norm: + { + file: (.File // .file // .Target // .Location.File // "unknown"), + line: (.StartLine // .Line // .Location.StartLine // 0), + rule: (.RuleID // .Rule // .Description // "unknown"), + commit: (.Commit // .commit // "") + }; + (if type=="object" and has("findings") then .findings + elif type=="array" then . + else [] end)[] | norm + | "โ€ข [\(.rule)] \(.file):\(.line) \(.commit[0:7] // "no-commit") '$CI_PROJECT_URL'/blob/\(.commit)/\(.file)#L\(.line)" + ' gitleaks.json | head -n 200 + + echo "" + echo "DEBUG: Finished displaying leaks" + + # Fail the job if leaks were found + if [[ "$COUNT" -gt 0 ]]; then + echo "" + echo "โŒ Pipeline failed due to $COUNT leak(s). Review gitleaks.json artifact." + exit 1 + fi + else + echo "โœ… No leaks found." + fi + + after_script: + - echo "๐Ÿงน Cleaning up runner workspace..." + - rm -f "$HOME/.local/bin/gitleaks" gitleaks.tgz || true + + artifacts: + when: always + paths: + - gitleaks.json + + allow_failure: false + +gitleaks_diff: + extends: .gitleaks_scan + variables: + SCAN_MODE: "diff" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + +gitleaks_full_manual: + extends: .gitleaks_scan + variables: + SCAN_MODE: "full" + rules: + - if: '$CI_PIPELINE_SOURCE == "web"' + +gitleaks_full_scheduled: + extends: .gitleaks_scan + variables: + SCAN_MODE: "full" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + From b6a31514257c73e59d0f39fece80b387c4a000d4 Mon Sep 17 00:00:00 2001 From: Vladimir Portnov Date: Wed, 22 Oct 2025 16:13:20 +0800 Subject: [PATCH 26/27] Use globally installed trdl when available (#52) Signed-off-by: Vladimir Portnov --- templates/Setup.gitlab-ci.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/templates/Setup.gitlab-ci.yml b/templates/Setup.gitlab-ci.yml index 3626ef2..9b8e373 100644 --- a/templates/Setup.gitlab-ci.yml +++ b/templates/Setup.gitlab-ci.yml @@ -32,14 +32,15 @@ before_script: echo "MODULES_MODULE_TAG: ${MODULES_MODULE_TAG}" # Setup trdl - | - echo "=== Setup trdl ===" - export PATH=$PATH:~/bin - trdl_target_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) - trdl_installed_version=$(trdl | tail -n 1 | sed -E 's/Version: v([0-9\.]+)/\1/') - if [[ "$trdl_installed_version" != "$trdl_target_version" ]]; then - curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_target_version/linux-amd64/bin/trdl" - install -D trdl ~/bin/trdl - rm trdl + if ! command -v trdl; then + export PATH=$PATH:~/bin + trdl_target_version=$(curl -s https://tuf.trdl.dev/targets/channels/0/stable) + trdl_installed_version=$(trdl | tail -n 1 | sed -E 's/Version: v([0-9\.]+)/\1/') + if [[ "$trdl_installed_version" != "$trdl_target_version" ]]; then + curl -sSLO "https://tuf.trdl.dev/targets/releases/$trdl_target_version/linux-amd64/bin/trdl" + install -D trdl ~/bin/trdl + rm trdl + fi fi # Setup werf From 39409cb838980749e1c6f4b0af242d888f1a85ce Mon Sep 17 00:00:00 2001 From: rtrofimenkov-ssdlc Date: Fri, 24 Oct 2025 15:56:29 +0500 Subject: [PATCH 27/27] Secret scan adjustments (#53) * refactor diff scan Signed-off-by: Roman Trofimenkov * refactor diff scan Signed-off-by: Roman Trofimenkov * fix base commit Signed-off-by: Roman Trofimenkov * format fix Signed-off-by: Roman Trofimenkov --------- Signed-off-by: Roman Trofimenkov --- templates/gitleaks.gitlab-ci.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/templates/gitleaks.gitlab-ci.yml b/templates/gitleaks.gitlab-ci.yml index 74fc069..8562fd0 100644 --- a/templates/gitleaks.gitlab-ci.yml +++ b/templates/gitleaks.gitlab-ci.yml @@ -3,6 +3,8 @@ variables: .gitleaks_scan: stage: gitleaks + variables: + GIT_DEPTH: 0 before_script: [] script: - | @@ -41,14 +43,16 @@ variables: GITLEAKS_EXIT=0 if [[ "$SCAN_MODE" == "diff" ]]; then echo "๐Ÿ•ต๏ธ Running in DIFF mode..." - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" --depth=1 - BASE_SHA=$(git merge-base origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME HEAD || echo "") - if [[ -z "$BASE_SHA" ]]; then - echo "โŒ BASE_SHA not found. Aborting." - exit 1 - fi - echo "โ–ถ Scanning diff: $BASE_SHA...HEAD" - gitleaks detect --no-banner --report-format json --report-path gitleaks.json $CONFIG_ARG --source . --log-opts "$BASE_SHA...HEAD" || GITLEAKS_EXIT=$? + + BASE_COMMIT=$(git merge-base "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" "$CI_COMMIT_SHA") + HEAD_COMMIT="$CI_COMMIT_SHA" + + echo "โ–ถ Target branch: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + echo "โ–ถ Base commit (merge-base): $BASE_COMMIT" + echo "โ–ถ Head commit: $HEAD_COMMIT" + echo "โ–ถ Scanning range: ${BASE_COMMIT}..${HEAD_COMMIT}" + + gitleaks detect --no-banner --report-format json --report-path gitleaks.json $CONFIG_ARG --source . --log-opts "${BASE_COMMIT}..${HEAD_COMMIT}" || GITLEAKS_EXIT=$? elif [[ "$SCAN_MODE" == "full" ]]; then echo "๐Ÿ•ต๏ธ Running in FULL mode..." gitleaks detect --no-banner --report-format json --report-path gitleaks.json $CONFIG_ARG --source . || GITLEAKS_EXIT=$?