From 8f1a23f9fd78e9d227bcc59ed43f571614151a63 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Thu, 12 Sep 2024 19:10:59 +0300 Subject: [PATCH 001/220] Redis CE 8.0 Milestone 1 release for Debian - Change the repository structure of the project to support usage of release branches. - Save the legacy templating scripts in a separate directory. - Add a GitHub Actions workflow that builds and tests the Docker images. https://github.com/redis/docker-library-redis/pull/409 --- .dockerignore | 3 + .gitattributes | 1 - .../actions/build-and-tag-locally/action.yml | 211 ++++++++++++++++++ .github/workflows/pre-merge.yml | 34 +++ .gitignore | 2 +- README.md | 6 +- alpine/Dockerfile | 154 +++++++++++++ alpine/docker-entrypoint.sh | 34 +++ debian/Dockerfile | 167 ++++++++++++++ debian/docker-entrypoint.sh | 34 +++ .../.github}/workflows/ci.yml | 0 .../.github}/workflows/verify-templating.yml | 0 .../6.2}/alpine/Dockerfile | 0 .../6.2}/alpine/docker-entrypoint.sh | 0 .../6.2}/debian/Dockerfile | 0 .../6.2}/debian/docker-entrypoint.sh | 0 .../7.0}/alpine/Dockerfile | 0 .../7.0}/alpine/docker-entrypoint.sh | 0 .../7.0}/debian/Dockerfile | 0 .../7.0}/debian/docker-entrypoint.sh | 0 .../7.2}/alpine/Dockerfile | 0 .../7.2}/alpine/docker-entrypoint.sh | 0 .../7.2}/debian/Dockerfile | 0 .../7.2}/debian/docker-entrypoint.sh | 0 .../7.4-rc}/alpine/Dockerfile | 0 .../7.4-rc}/alpine/docker-entrypoint.sh | 0 .../7.4-rc}/debian/Dockerfile | 0 .../7.4-rc}/debian/docker-entrypoint.sh | 0 .../7.4}/alpine/Dockerfile | 0 .../7.4}/alpine/docker-entrypoint.sh | 0 .../7.4}/debian/Dockerfile | 0 .../7.4}/debian/docker-entrypoint.sh | 0 .../BUILD.md | 0 .../Dockerfile.template | 0 .../apply-templates.sh | 0 .../docker-entrypoint.sh | 0 .../generate-stackbrew-library-mac.sh | 0 .../generate-stackbrew-library.sh | 0 .../stackbrew-generated}/7-4.txt | 0 .../update.sh | 0 .../versions.json | 0 .../versions.sh | 0 42 files changed, 640 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/actions/build-and-tag-locally/action.yml create mode 100644 .github/workflows/pre-merge.yml create mode 100644 alpine/Dockerfile create mode 100755 alpine/docker-entrypoint.sh create mode 100644 debian/Dockerfile create mode 100755 debian/docker-entrypoint.sh rename {.github => legacy-templating-scripts/.github}/workflows/ci.yml (100%) rename {.github => legacy-templating-scripts/.github}/workflows/verify-templating.yml (100%) rename {6.2 => legacy-templating-scripts/6.2}/alpine/Dockerfile (100%) rename {6.2 => legacy-templating-scripts/6.2}/alpine/docker-entrypoint.sh (100%) rename {6.2 => legacy-templating-scripts/6.2}/debian/Dockerfile (100%) rename {6.2 => legacy-templating-scripts/6.2}/debian/docker-entrypoint.sh (100%) rename {7.0 => legacy-templating-scripts/7.0}/alpine/Dockerfile (100%) rename {7.0 => legacy-templating-scripts/7.0}/alpine/docker-entrypoint.sh (100%) rename {7.0 => legacy-templating-scripts/7.0}/debian/Dockerfile (100%) rename {7.0 => legacy-templating-scripts/7.0}/debian/docker-entrypoint.sh (100%) rename {7.2 => legacy-templating-scripts/7.2}/alpine/Dockerfile (100%) rename {7.2 => legacy-templating-scripts/7.2}/alpine/docker-entrypoint.sh (100%) rename {7.2 => legacy-templating-scripts/7.2}/debian/Dockerfile (100%) rename {7.2 => legacy-templating-scripts/7.2}/debian/docker-entrypoint.sh (100%) rename {7.4-rc => legacy-templating-scripts/7.4-rc}/alpine/Dockerfile (100%) rename {7.4-rc => legacy-templating-scripts/7.4-rc}/alpine/docker-entrypoint.sh (100%) rename {7.4-rc => legacy-templating-scripts/7.4-rc}/debian/Dockerfile (100%) rename {7.4-rc => legacy-templating-scripts/7.4-rc}/debian/docker-entrypoint.sh (100%) rename {7.4 => legacy-templating-scripts/7.4}/alpine/Dockerfile (100%) rename {7.4 => legacy-templating-scripts/7.4}/alpine/docker-entrypoint.sh (100%) rename {7.4 => legacy-templating-scripts/7.4}/debian/Dockerfile (100%) rename {7.4 => legacy-templating-scripts/7.4}/debian/docker-entrypoint.sh (100%) rename BUILD.md => legacy-templating-scripts/BUILD.md (100%) rename Dockerfile.template => legacy-templating-scripts/Dockerfile.template (100%) rename apply-templates.sh => legacy-templating-scripts/apply-templates.sh (100%) rename docker-entrypoint.sh => legacy-templating-scripts/docker-entrypoint.sh (100%) rename generate-stackbrew-library-mac.sh => legacy-templating-scripts/generate-stackbrew-library-mac.sh (100%) rename generate-stackbrew-library.sh => legacy-templating-scripts/generate-stackbrew-library.sh (100%) rename {stackbrew-generated => legacy-templating-scripts/stackbrew-generated}/7-4.txt (100%) rename update.sh => legacy-templating-scripts/update.sh (100%) rename versions.json => legacy-templating-scripts/versions.json (100%) rename versions.sh => legacy-templating-scripts/versions.sh (100%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..45060a522 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.github/** +.git/** +.git* diff --git a/.gitattributes b/.gitattributes index 14a112269..778f645e2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,2 @@ /*/**/Dockerfile linguist-generated /*/**/docker-entrypoint.sh linguist-generated -/Dockerfile*.template linguist-language=Dockerfile diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml new file mode 100644 index 000000000..149661d77 --- /dev/null +++ b/.github/actions/build-and-tag-locally/action.yml @@ -0,0 +1,211 @@ +name: Build and Test + +inputs: + distribution: + description: "Distribution flavor" + default: "debian" + platform: + description: "Platform" + required: true + publish_image: + description: "Publish image to Docker Hub" + default: "false" + registry_username: + description: "Docker Hub username" + required: false + registry_password: + description: "Docker Hub password" + required: false + registry_repository: + description: 'Repository to push the image to' + required: false + +runs: + using: "composite" + steps: + - name: Install QEMU + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Calculate architecture name + id: platform + shell: bash + run: | + case ${{ inputs.platform }} in + linux/amd64) + plaform_name="amd64" + ;; + linux/arm64) + plaform_name="arm64" + ;; + linux/arm/v5) + plaform_name="arm-v5" + ;; + linux/arm/v7) + plaform_name="arm-v7" + ;; + linux/i386) + plaform_name="i386" + ;; + linux/mips64le) + plaform_name="mips64le" + ;; + linux/ppc64le) + plaform_name="ppc64le" + ;; + linux/s390x) + plaform_name="s390x" + ;; + *) + echo "Architecture not supported: ${{ inputs.platform }}" + exit 1 + ;; + esac + echo "display_name=$plaform_name" >> "$GITHUB_OUTPUT" + + - name: Clean up + shell: bash + run: | + docker rm -f sanity-test-${{ steps.platform.outputs.display_name }} || true + docker rmi -f ${{ github.sha }}:${{ steps.platform.outputs.display_name }} || true + + - name: Docker Login + uses: docker/login-action@v3 + if: inputs.publish_image == 'true' + with: + registry: ${{ inputs.registry_repository }} + username: ${{ inputs.registry_username }} + password: ${{ inputs.registry_password }} + + - name: Build + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ inputs.distribution }}/Dockerfile + push: false + load: true + platforms: ${{ inputs.platform }} + tags: ${{ github.sha }}:${{ steps.platform.outputs.display_name }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Save image + shell: bash + run: | + docker save -o /tmp/image-${{ steps.platform.outputs.display_name }}.tar ${{ github.sha }}:${{ steps.platform.outputs.display_name }} + + - name: Upload image + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.platform.outputs.display_name }}-docker-image.tar + path: /tmp/image-${{ steps.platform.outputs.display_name }}.tar + retention-days: 45 + + - name: Run container + shell: bash + if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + run: | + docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} + + - name: Container Logs + if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker logs sanity-test-${{ steps.platform.outputs.display_name }} + + - name: Sanity Tests + if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server + + - name: Verify installed modules + if: ${{ contains(fromJSON('["amd64", "arm64",]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + modules=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli module list) + echo "Installed modules:" + echo "$modules" + missing_modules=() + for module in "bf" "search" "timeseries" "ReJSON"; do + if ! echo "$modules" | grep -q "$module"; then + missing_modules+=("$module") + fi + done + if [ ${#missing_modules[@]} -eq 0 ]; then + echo "All required modules are installed" + else + echo "The following modules are missing: ${missing_modules[*]}" + exit 1 + fi + + - name: Test RedisBloom + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.ADD popular_keys "redis:hash" + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.ADD popular_keys "redis:set" + [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:hash")" = "1" ] || { echo "RedisBloom test failed: 'redis:hash' not found"; exit 1; } + [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:list")" = "0" ] || { echo "RedisBloom test failed: 'redis:list' found unexpectedly"; exit 1; } + echo "RedisBloom test passed successfully" + + - name: Test RediSearch + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli FT.CREATE redis_commands ON HASH PREFIX 1 cmd: SCHEMA name TEXT SORTABLE description TEXT + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli HSET cmd:set name "SET" description "Set the string value of a key" + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli HSET cmd:get name "GET" description "Get the value of a key" + result=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli FT.SEARCH redis_commands "value") + if echo "$result" | grep -q "Set the string value of a key" && echo "$result" | grep -q "Get the value of a key"; then + echo "RediSearch test passed successfully" + else + echo "RediSearch test failed: expected commands not found in search results" + exit 1 + fi + + - name: Test RedisTimeSeries + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.CREATE redis:cpu:usage RETENTION 86400 + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.ADD redis:cpu:usage "*" 80 + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.ADD redis:cpu:usage "*" 65 + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.ADD redis:cpu:usage "*" 70 + result=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.RANGE redis:cpu:usage - + COUNT 3) + if echo "$result" | grep -q "80" && echo "$result" | grep -q "65" && echo "$result" | grep -q "70"; then + echo "RedisTimeSeries test passed successfully" + else + echo "RedisTimeSeries test failed: expected values not found in time series" + exit 1 + fi + + - name: Test ReJSON + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + shell: bash + run: | + docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli JSON.SET redis:config $ '{"maxmemory":"2gb","maxmemory-policy":"allkeys-lru"}' + result=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli JSON.GET redis:config $.maxmemory-policy) + cleaned_result=$(echo $result | tr -d '[]"') + if [ "$cleaned_result" = "allkeys-lru" ]; then + echo "ReJSON test passed successfully" + else + echo "ReJSON test failed: expected 'allkeys-lru', got $result" + exit 1 + fi + + - name: Push image + uses: docker/build-push-action@v6 + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64","arm64"]'), steps.platform.outputs.display_name) }} + with: + context: . + file: ${{ inputs.distribution }}/Dockerfile + push: true + tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml new file mode 100644 index 000000000..53e3d0512 --- /dev/null +++ b/.github/workflows/pre-merge.yml @@ -0,0 +1,34 @@ +name: Build and Test +on: + pull_request: + branches: + - master + - release/* + +jobs: + build-and-test: + runs-on: [ubuntu-latest] + strategy: + matrix: + distribution: + - debian + #- alpine + platform: + - linux/amd64 + - linux/i386 + - linux/arm/v5 + - linux/arm/v7 + - linux/mips64le + - linux/ppc64le + - linux/s390x + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: ./.github/actions/build-and-tag-locally + with: + distribution: ${{ matrix.distribution }} + platform: ${{ matrix.platform }} + registry_username: ${{ vars.REGISTRY_USERNAME }} + registry_password: ${{ secrets.REGISTRY_PASSWORD }} + publish_image: ${{ vars.PUBLISH_IMAGE }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY }} diff --git a/.gitignore b/.gitignore index d548f66de..e43b0f988 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.jq-template.awk +.DS_Store diff --git a/README.md b/README.md index b3dea9433..27af22886 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # https://github.com/docker-library/redis -## Maintained by: [the Docker Community](https://github.com/docker-library/redis) +## Maintained by: [Redis LTD](https://redis.io/) -This is the Git repo of the [Docker "Official Image"](https://github.com/docker-library/official-images#what-are-official-images) for [`redis`](https://hub.docker.com/_/redis/) (not to be confused with any official `redis` image provided by `redis` upstream). See [the Docker Hub page](https://hub.docker.com/_/redis/) for the full readme on how to use this Docker image and for information regarding contributing and issues. +This is the Git repo of the [Docker "Official Image"](https://github.com/docker-library/official-images#what-are-official-images) for [`redis`](https://hub.docker.com/_/redis/). See [the Docker Hub page](https://hub.docker.com/_/redis/) for the full `README` on how to use this Docker image and for information regarding contributing and issues. The [full image description on Docker Hub](https://hub.docker.com/_/redis/) is generated/maintained over in [the docker-library/docs repository](https://github.com/docker-library/docs), specifically in [the `redis` directory](https://github.com/docker-library/docs/tree/master/redis). @@ -22,5 +22,3 @@ For outstanding `redis` image PRs, check [PRs with the "library/redis" label on | [![amd64 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/amd64/job/redis.svg?label=amd64)](https://doi-janky.infosiftr.net/job/multiarch/job/amd64/job/redis/) | [![arm32v5 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v5/job/redis.svg?label=arm32v5)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v5/job/redis/) | [![arm32v6 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v6/job/redis.svg?label=arm32v6)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v6/job/redis/) | [![arm32v7 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm32v7/job/redis.svg?label=arm32v7)](https://doi-janky.infosiftr.net/job/multiarch/job/arm32v7/job/redis/) | | [![arm64v8 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/arm64v8/job/redis.svg?label=arm64v8)](https://doi-janky.infosiftr.net/job/multiarch/job/arm64v8/job/redis/) | [![i386 build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/i386/job/redis.svg?label=i386)](https://doi-janky.infosiftr.net/job/multiarch/job/i386/job/redis/) | [![mips64le build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/mips64le/job/redis.svg?label=mips64le)](https://doi-janky.infosiftr.net/job/multiarch/job/mips64le/job/redis/) | [![ppc64le build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/ppc64le/job/redis.svg?label=ppc64le)](https://doi-janky.infosiftr.net/job/multiarch/job/ppc64le/job/redis/) | | [![s390x build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/multiarch/job/s390x/job/redis.svg?label=s390x)](https://doi-janky.infosiftr.net/job/multiarch/job/s390x/job/redis/) | [![put-shared build status badge](https://img.shields.io/jenkins/s/https/doi-janky.infosiftr.net/job/put-shared/job/light/job/redis.svg?label=put-shared)](https://doi-janky.infosiftr.net/job/put-shared/job/light/job/redis/) | - - diff --git a/alpine/Dockerfile b/alpine/Dockerfile new file mode 100644 index 000000000..9c2a3747e --- /dev/null +++ b/alpine/Dockerfile @@ -0,0 +1,154 @@ +ARG BASE_IMAGE=alpine:3.20 +FROM ${BASE_IMAGE} AS sources +ARG REDIS_VERSION=8.0-m01 +ARG REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/${REDIS_VERSION}.tar.gz +ARG REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c +WORKDIR /files +ADD --checksum=sha256:${REDIS_DOWNLOAD_SHA} ${REDIS_DOWNLOAD_URL} . +RUN tar -xvf *.tar.gz && mv redis-${REDIS_VERSION} redis + +FROM ${BASE_IMAGE} + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN set -eux; \ +# alpine already has a gid 999, so we'll use the next id + addgroup -S -g 1000 redis; \ + adduser -S -G redis -u 999 redis + +# runtime dependencies +RUN set -eux; \ + apk add --no-cache \ +# add tzdata for https://github.com/docker-library/redis/issues/138 + tzdata \ + ; + +# grab gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION=1.17 +RUN set -eux; \ + apk add --no-cache --virtual .gosu-fetch gnupg; \ + arch="$(apk --print-arch)"; \ + case "$arch" in \ + 'x86_64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-amd64'; sha256='bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3' ;; \ + 'aarch64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-arm64'; sha256='c3805a85d17f4454c23d7059bcb97e1ec1af272b90126e79ed002342de08389b' ;; \ + 'armhf') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ + 'x86') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-i386'; sha256='087dbb8fe479537e64f9c86fa49ff3b41dee1cbd28739a19aaef83dc8186b1ca' ;; \ + 'ppc64le') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-ppc64el'; sha256='1891acdcfa70046818ab6ed3c52b9d42fa10fbb7b340eb429c8c7849691dbd76' ;; \ + 'riscv64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-riscv64'; sha256='38a6444b57adce135c42d5a3689f616fc7803ddc7a07ff6f946f2ebc67a26ba6' ;; \ + 's390x') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-s390x'; sha256='69873bab588192f760547ca1f75b27cfcf106e9f7403fee6fd0600bc914979d0' ;; \ + 'armv7') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ + *) echo >&2 "error: unsupported gosu architecture: '$arch'"; exit 1 ;; \ + esac; \ + wget -O /usr/local/bin/gosu.asc "$url.asc"; \ + wget -O /usr/local/bin/gosu "$url"; \ + echo "$sha256 */usr/local/bin/gosu" | sha256sum -c -; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -f "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apk del --no-network .gosu-fetch; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw set -eux; \ + \ + apk add --no-cache --virtual .build-deps \ + coreutils \ + dpkg-dev dpkg \ + gcc \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + ; \ + \ + arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + case "$arch" in \ + 'amd64') export BUILD_WITH_MODULES=yes; export INSTALL_RUST_TOOLCHAIN=yes; export DISABLE_WERRORS=yes ;; \ + 'arm64') export BUILD_WITH_MODULES=yes; export INSTALL_RUST_TOOLCHAIN=yes; export DISABLE_WERRORS=yes ;; \ + *) echo >&2 "Modules are NOT supported! unsupported architecture: '$arch'"; export BUILD_WITH_MODULES=no ;; \ + esac; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + apk add --no-cache --virtual .module-build-deps \ + git \ + cmake \ + curl \ + python3 \ + py3-pip \ + py3-virtualenv \ + python3-dev \ + unzip \ + rsync \ + clang \ + automake \ + autoconf \ + libtool \ + g++ \ + libgcc \ + ;\ + fi; \ + \ +# disable Redis protected mode [1] as it is unnecessary in context of Docker +# (ports are not automatically exposed when running inside Docker, but rather explicitly by specifying -p / -P) +# [1]: https://github.com/redis/redis/commit/edd4d555df57dc84265fdfb4ef59a4678832f6da + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' /usr/src/redis/src/config.c; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' /usr/src/redis/src/config.c; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' /usr/src/redis/src/config.c; \ +# for future reference, we modify this directly in the source instead of just supplying a default configuration flag because apparently "if you specify any argument to redis-server, [it assumes] you are going to specify everything" +# see also https://github.com/docker-library/redis/issues/4#issuecomment-50780840 +# (more exactly, this makes sure the default behavior of "save on SIGTERM" stays functional by default) + \ +# https://github.com/jemalloc/jemalloc/issues/467 -- we need to patch the "./configure" for the bundled jemalloc to match how Debian compiles, for compatibility +# (also, we do cross-builds, so we need to embed the appropriate "--build=xxx" values to that "./configure" invocation) + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + extraJemallocConfigureFlags="--build=$gnuArch"; \ +# https://salsa.debian.org/debian/jemalloc/-/blob/c0a88c37a551be7d12e4863435365c9a6a51525f/debian/rules#L8-23 + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ + amd64 | i386 | x32) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=12" ;; \ + *) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=16" ;; \ + esac; \ + extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-hugepage=21"; \ + grep -F 'cd jemalloc && ./configure ' /usr/src/redis/deps/Makefile; \ + sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \ + grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ + \ + export BUILD_TLS=yes; \ + make -C /usr/src/redis -j "$(nproc)" all; \ + make -C /usr/src/redis install; \ + \ +# TODO https://github.com/redis/redis/pull/3494 (deduplicate "redis-server" copies) + serverMd5="$(md5sum /usr/local/bin/redis-server | cut -d' ' -f1)"; export serverMd5; \ + find /usr/local/bin/redis* -maxdepth 0 \ + -type f -not -name redis-server \ + -exec sh -eux -c ' \ + md5="$(md5sum "$1" | cut -d" " -f1)"; \ + test "$md5" = "$serverMd5"; \ + ' -- '{}' ';' \ + -exec ln -svfT 'redis-server' '{}' ';' \ + ; \ + \ + make -C /usr/src/redis distclean; \ + \ + runDeps="$( \ + scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \ + | tr ',' '\n' \ + | sort -u \ + | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ + )"; \ + apk add --no-network --virtual .redis-rundeps $runDeps; \ + apk del --no-network .build-deps; \ + \ + redis-cli --version; \ + redis-server --version; +RUN mkdir /data && chown redis:redis /data +VOLUME /data +WORKDIR /data + +COPY alpine/docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 6379 +CMD ["redis-server"] diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh new file mode 100755 index 000000000..77e274f96 --- /dev/null +++ b/alpine/docker-entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +# first arg is `-f` or `--some-option` +# or first arg is `something.conf` +if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then + set -- redis-server "$@" +fi + +# allow the container to be started with `--user` +if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then + find . \! -user redis -exec chown redis '{}' + + exec gosu redis "$0" "$@" +fi + +# set an appropriate umask (if one isn't set already) +# - https://github.com/docker-library/redis/issues/305 +# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37 +um="$(umask)" +if [ "$um" = '0022' ]; then + umask 0077 +fi + +modules_dir="/usr/local/lib/redis/modules/" +command="exec \"\$@\"" + +if [ -n "$(ls -A $modules_dir)" ]; then + for module in "$modules_dir"/*.so; + do + command="$command --loadmodule $module" + done +fi + +eval "$command" \ No newline at end of file diff --git a/debian/Dockerfile b/debian/Dockerfile new file mode 100644 index 000000000..9b57bb6e5 --- /dev/null +++ b/debian/Dockerfile @@ -0,0 +1,167 @@ +ARG BASE_IMAGE=debian:bookworm-slim +FROM ${BASE_IMAGE} AS sources +ARG REDIS_VERSION=8.0-m01 +ARG REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/${REDIS_VERSION}.tar.gz +ARG REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c +WORKDIR /files +ADD --checksum=sha256:${REDIS_DOWNLOAD_SHA} ${REDIS_DOWNLOAD_URL} . +RUN apt-get update && apt-get install -y tar patch +RUN tar -xvf *.tar.gz && mv redis-${REDIS_VERSION} redis + +FROM ${BASE_IMAGE} + +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN set -eux; \ + groupadd -r -g 999 redis; \ + useradd -r -g redis -u 999 redis + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ +# add tzdata explicitly for https://github.com/docker-library/redis/issues/138 (see also https://bugs.debian.org/837060 and related) + tzdata \ + ; \ + rm -rf /var/lib/apt/lists/* + +# grab gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION=1.17 +RUN set -eux; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates gnupg wget; \ + rm -rf /var/lib/apt/lists/*; \ + arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + case "$arch" in \ + 'amd64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-amd64'; sha256='bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3' ;; \ + 'arm64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-arm64'; sha256='c3805a85d17f4454c23d7059bcb97e1ec1af272b90126e79ed002342de08389b' ;; \ + 'armel') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armel'; sha256='f9969910fa141140438c998cfa02f603bf213b11afd466dcde8fa940e700945d' ;; \ + 'i386') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-i386'; sha256='087dbb8fe479537e64f9c86fa49ff3b41dee1cbd28739a19aaef83dc8186b1ca' ;; \ + 'mips64el') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-mips64el'; sha256='87140029d792595e660be0015341dfa1c02d1181459ae40df9f093e471d75b70' ;; \ + 'ppc64el') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-ppc64el'; sha256='1891acdcfa70046818ab6ed3c52b9d42fa10fbb7b340eb429c8c7849691dbd76' ;; \ + 'riscv64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-riscv64'; sha256='38a6444b57adce135c42d5a3689f616fc7803ddc7a07ff6f946f2ebc67a26ba6' ;; \ + 's390x') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-s390x'; sha256='69873bab588192f760547ca1f75b27cfcf106e9f7403fee6fd0600bc914979d0' ;; \ + 'armhf') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ + *) echo >&2 "error: unsupported gosu architecture: '$arch'"; exit 1 ;; \ + esac; \ + wget -O /usr/local/bin/gosu.asc "$url.asc"; \ + wget -O /usr/local/bin/gosu "$url"; \ + echo "$sha256 */usr/local/bin/gosu" | sha256sum -c -; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true + +RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + dpkg-dev \ + gcc \ + libc6-dev \ + libssl-dev \ + make \ + ; \ + \ + arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + case "$arch" in \ + 'amd64') export BUILD_WITH_MODULES=yes; export INSTALL_RUST_TOOLCHAIN=yes; export DISABLE_WERRORS=yes ;; \ + 'arm64') export BUILD_WITH_MODULES=yes; export INSTALL_RUST_TOOLCHAIN=yes; export DISABLE_WERRORS=yes ;; \ + *) echo >&2 "Modules are NOT supported! unsupported architecture: '$arch'"; export BUILD_WITH_MODULES=no ;; \ + esac; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + apt-get install -y --no-install-recommends \ + git \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + python3-dev \ + unzip \ + rsync \ + clang \ + automake \ + autoconf \ + libtool \ + g++; \ + fi; \ + \ + rm -rf /var/lib/apt/lists/*; \ + \ +# disable Redis protected mode [1] as it is unnecessary in context of Docker +# (ports are not automatically exposed when running inside Docker, but rather explicitly by specifying -p / -P) +# [1]: https://github.com/redis/redis/commit/edd4d555df57dc84265fdfb4ef59a4678832f6da + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' /usr/src/redis/src/config.c; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' /usr/src/redis/src/config.c; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' /usr/src/redis/src/config.c; \ +# for future reference, we modify this directly in the source instead of just supplying a default configuration flag because apparently "if you specify any argument to redis-server, [it assumes] you are going to specify everything" +# see also https://github.com/docker-library/redis/issues/4#issuecomment-50780840 +# (more exactly, this makes sure the default behavior of "save on SIGTERM" stays functional by default) + \ +# https://github.com/jemalloc/jemalloc/issues/467 -- we need to patch the "./configure" for the bundled jemalloc to match how Debian compiles, for compatibility +# (also, we do cross-builds, so we need to embed the appropriate "--build=xxx" values to that "./configure" invocation) + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + extraJemallocConfigureFlags="--build=$gnuArch"; \ +# https://salsa.debian.org/debian/jemalloc/-/blob/c0a88c37a551be7d12e4863435365c9a6a51525f/debian/rules#L8-23 + case "${arch##*-}" in \ + amd64 | i386 | x32) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=12" ;; \ + *) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=16" ;; \ + esac; \ + extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-hugepage=21"; \ + grep -F 'cd jemalloc && ./configure ' /usr/src/redis/deps/Makefile; \ + sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \ + grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ + \ + export BUILD_TLS=yes; \ + make -C /usr/src/redis -j "$(nproc)" all; \ + make -C /usr/src/redis install; \ + \ +# TODO https://github.com/redis/redis/pull/3494 (deduplicate "redis-server" copies) + serverMd5="$(md5sum /usr/local/bin/redis-server | cut -d' ' -f1)"; export serverMd5; \ + find /usr/local/bin/redis* -maxdepth 0 \ + -type f -not -name redis-server \ + -exec sh -eux -c ' \ + md5="$(md5sum "$1" | cut -d" " -f1)"; \ + test "$md5" = "$serverMd5"; \ + ' -- '{}' ';' \ + -exec ln -svfT 'redis-server' '{}' ';' \ + ; \ + \ + make -C /usr/src/redis distclean; \ + \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { so = $(NF-1); if (index(so, "/usr/local/") == 1) { next }; gsub("^/(usr/)?", "", so); printf "*%s\n", so }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + ; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/cache/debconf/*; \ + \ + redis-cli --version; \ + redis-server --version + +RUN mkdir /data && chown redis:redis /data +VOLUME /data +WORKDIR /data + +COPY debian/docker-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 6379 +CMD ["redis-server"] diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh new file mode 100755 index 000000000..77e274f96 --- /dev/null +++ b/debian/docker-entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +# first arg is `-f` or `--some-option` +# or first arg is `something.conf` +if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then + set -- redis-server "$@" +fi + +# allow the container to be started with `--user` +if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then + find . \! -user redis -exec chown redis '{}' + + exec gosu redis "$0" "$@" +fi + +# set an appropriate umask (if one isn't set already) +# - https://github.com/docker-library/redis/issues/305 +# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37 +um="$(umask)" +if [ "$um" = '0022' ]; then + umask 0077 +fi + +modules_dir="/usr/local/lib/redis/modules/" +command="exec \"\$@\"" + +if [ -n "$(ls -A $modules_dir)" ]; then + for module in "$modules_dir"/*.so; + do + command="$command --loadmodule $module" + done +fi + +eval "$command" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/legacy-templating-scripts/.github/workflows/ci.yml similarity index 100% rename from .github/workflows/ci.yml rename to legacy-templating-scripts/.github/workflows/ci.yml diff --git a/.github/workflows/verify-templating.yml b/legacy-templating-scripts/.github/workflows/verify-templating.yml similarity index 100% rename from .github/workflows/verify-templating.yml rename to legacy-templating-scripts/.github/workflows/verify-templating.yml diff --git a/6.2/alpine/Dockerfile b/legacy-templating-scripts/6.2/alpine/Dockerfile similarity index 100% rename from 6.2/alpine/Dockerfile rename to legacy-templating-scripts/6.2/alpine/Dockerfile diff --git a/6.2/alpine/docker-entrypoint.sh b/legacy-templating-scripts/6.2/alpine/docker-entrypoint.sh similarity index 100% rename from 6.2/alpine/docker-entrypoint.sh rename to legacy-templating-scripts/6.2/alpine/docker-entrypoint.sh diff --git a/6.2/debian/Dockerfile b/legacy-templating-scripts/6.2/debian/Dockerfile similarity index 100% rename from 6.2/debian/Dockerfile rename to legacy-templating-scripts/6.2/debian/Dockerfile diff --git a/6.2/debian/docker-entrypoint.sh b/legacy-templating-scripts/6.2/debian/docker-entrypoint.sh similarity index 100% rename from 6.2/debian/docker-entrypoint.sh rename to legacy-templating-scripts/6.2/debian/docker-entrypoint.sh diff --git a/7.0/alpine/Dockerfile b/legacy-templating-scripts/7.0/alpine/Dockerfile similarity index 100% rename from 7.0/alpine/Dockerfile rename to legacy-templating-scripts/7.0/alpine/Dockerfile diff --git a/7.0/alpine/docker-entrypoint.sh b/legacy-templating-scripts/7.0/alpine/docker-entrypoint.sh similarity index 100% rename from 7.0/alpine/docker-entrypoint.sh rename to legacy-templating-scripts/7.0/alpine/docker-entrypoint.sh diff --git a/7.0/debian/Dockerfile b/legacy-templating-scripts/7.0/debian/Dockerfile similarity index 100% rename from 7.0/debian/Dockerfile rename to legacy-templating-scripts/7.0/debian/Dockerfile diff --git a/7.0/debian/docker-entrypoint.sh b/legacy-templating-scripts/7.0/debian/docker-entrypoint.sh similarity index 100% rename from 7.0/debian/docker-entrypoint.sh rename to legacy-templating-scripts/7.0/debian/docker-entrypoint.sh diff --git a/7.2/alpine/Dockerfile b/legacy-templating-scripts/7.2/alpine/Dockerfile similarity index 100% rename from 7.2/alpine/Dockerfile rename to legacy-templating-scripts/7.2/alpine/Dockerfile diff --git a/7.2/alpine/docker-entrypoint.sh b/legacy-templating-scripts/7.2/alpine/docker-entrypoint.sh similarity index 100% rename from 7.2/alpine/docker-entrypoint.sh rename to legacy-templating-scripts/7.2/alpine/docker-entrypoint.sh diff --git a/7.2/debian/Dockerfile b/legacy-templating-scripts/7.2/debian/Dockerfile similarity index 100% rename from 7.2/debian/Dockerfile rename to legacy-templating-scripts/7.2/debian/Dockerfile diff --git a/7.2/debian/docker-entrypoint.sh b/legacy-templating-scripts/7.2/debian/docker-entrypoint.sh similarity index 100% rename from 7.2/debian/docker-entrypoint.sh rename to legacy-templating-scripts/7.2/debian/docker-entrypoint.sh diff --git a/7.4-rc/alpine/Dockerfile b/legacy-templating-scripts/7.4-rc/alpine/Dockerfile similarity index 100% rename from 7.4-rc/alpine/Dockerfile rename to legacy-templating-scripts/7.4-rc/alpine/Dockerfile diff --git a/7.4-rc/alpine/docker-entrypoint.sh b/legacy-templating-scripts/7.4-rc/alpine/docker-entrypoint.sh similarity index 100% rename from 7.4-rc/alpine/docker-entrypoint.sh rename to legacy-templating-scripts/7.4-rc/alpine/docker-entrypoint.sh diff --git a/7.4-rc/debian/Dockerfile b/legacy-templating-scripts/7.4-rc/debian/Dockerfile similarity index 100% rename from 7.4-rc/debian/Dockerfile rename to legacy-templating-scripts/7.4-rc/debian/Dockerfile diff --git a/7.4-rc/debian/docker-entrypoint.sh b/legacy-templating-scripts/7.4-rc/debian/docker-entrypoint.sh similarity index 100% rename from 7.4-rc/debian/docker-entrypoint.sh rename to legacy-templating-scripts/7.4-rc/debian/docker-entrypoint.sh diff --git a/7.4/alpine/Dockerfile b/legacy-templating-scripts/7.4/alpine/Dockerfile similarity index 100% rename from 7.4/alpine/Dockerfile rename to legacy-templating-scripts/7.4/alpine/Dockerfile diff --git a/7.4/alpine/docker-entrypoint.sh b/legacy-templating-scripts/7.4/alpine/docker-entrypoint.sh similarity index 100% rename from 7.4/alpine/docker-entrypoint.sh rename to legacy-templating-scripts/7.4/alpine/docker-entrypoint.sh diff --git a/7.4/debian/Dockerfile b/legacy-templating-scripts/7.4/debian/Dockerfile similarity index 100% rename from 7.4/debian/Dockerfile rename to legacy-templating-scripts/7.4/debian/Dockerfile diff --git a/7.4/debian/docker-entrypoint.sh b/legacy-templating-scripts/7.4/debian/docker-entrypoint.sh similarity index 100% rename from 7.4/debian/docker-entrypoint.sh rename to legacy-templating-scripts/7.4/debian/docker-entrypoint.sh diff --git a/BUILD.md b/legacy-templating-scripts/BUILD.md similarity index 100% rename from BUILD.md rename to legacy-templating-scripts/BUILD.md diff --git a/Dockerfile.template b/legacy-templating-scripts/Dockerfile.template similarity index 100% rename from Dockerfile.template rename to legacy-templating-scripts/Dockerfile.template diff --git a/apply-templates.sh b/legacy-templating-scripts/apply-templates.sh similarity index 100% rename from apply-templates.sh rename to legacy-templating-scripts/apply-templates.sh diff --git a/docker-entrypoint.sh b/legacy-templating-scripts/docker-entrypoint.sh similarity index 100% rename from docker-entrypoint.sh rename to legacy-templating-scripts/docker-entrypoint.sh diff --git a/generate-stackbrew-library-mac.sh b/legacy-templating-scripts/generate-stackbrew-library-mac.sh similarity index 100% rename from generate-stackbrew-library-mac.sh rename to legacy-templating-scripts/generate-stackbrew-library-mac.sh diff --git a/generate-stackbrew-library.sh b/legacy-templating-scripts/generate-stackbrew-library.sh similarity index 100% rename from generate-stackbrew-library.sh rename to legacy-templating-scripts/generate-stackbrew-library.sh diff --git a/stackbrew-generated/7-4.txt b/legacy-templating-scripts/stackbrew-generated/7-4.txt similarity index 100% rename from stackbrew-generated/7-4.txt rename to legacy-templating-scripts/stackbrew-generated/7-4.txt diff --git a/update.sh b/legacy-templating-scripts/update.sh similarity index 100% rename from update.sh rename to legacy-templating-scripts/update.sh diff --git a/versions.json b/legacy-templating-scripts/versions.json similarity index 100% rename from versions.json rename to legacy-templating-scripts/versions.json diff --git a/versions.sh b/legacy-templating-scripts/versions.sh similarity index 100% rename from versions.sh rename to legacy-templating-scripts/versions.sh From 1f67149f2ee7a924cf50ad981c1672c54cf06a48 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Fri, 13 Sep 2024 10:40:09 +0300 Subject: [PATCH 002/220] Comply with Docker official-images constraints (#410) - Remove usage of `ARG`, and replace with `ENV`. - Remove usage ADD of remote resources. https://github.com/docker-library/official-images/pull/17549#issuecomment-2347176071 --- alpine/Dockerfile | 16 ++++++++-------- debian/Dockerfile | 17 ++++++++--------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 9c2a3747e..7ae463e80 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -1,13 +1,13 @@ -ARG BASE_IMAGE=alpine:3.20 -FROM ${BASE_IMAGE} AS sources -ARG REDIS_VERSION=8.0-m01 -ARG REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/${REDIS_VERSION}.tar.gz -ARG REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c +FROM alpine:3.20 AS sources +ENV REDIS_VERSION=8.0-m01 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c WORKDIR /files -ADD --checksum=sha256:${REDIS_DOWNLOAD_SHA} ${REDIS_DOWNLOAD_URL} . -RUN tar -xvf *.tar.gz && mv redis-${REDIS_VERSION} redis +RUN apk add tar patch wget +RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; echo "$REDIS_DOWNLOAD_SHA redis.tar.gz" | sha256sum -c -; +RUN tar -xvf redis.tar.gz && mv redis-$REDIS_VERSION redis -FROM ${BASE_IMAGE} +FROM alpine:3.20 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added RUN set -eux; \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 9b57bb6e5..20c3d7d79 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -1,14 +1,13 @@ -ARG BASE_IMAGE=debian:bookworm-slim -FROM ${BASE_IMAGE} AS sources -ARG REDIS_VERSION=8.0-m01 -ARG REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/${REDIS_VERSION}.tar.gz -ARG REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c +FROM debian:bookworm-slim AS sources +ENV REDIS_VERSION=8.0-m01 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c WORKDIR /files -ADD --checksum=sha256:${REDIS_DOWNLOAD_SHA} ${REDIS_DOWNLOAD_URL} . -RUN apt-get update && apt-get install -y tar patch -RUN tar -xvf *.tar.gz && mv redis-${REDIS_VERSION} redis +RUN apt-get update && apt-get install -y tar patch wget +RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; echo "$REDIS_DOWNLOAD_SHA redis.tar.gz" | sha256sum -c -; +RUN tar -xvf redis.tar.gz && mv redis-$REDIS_VERSION redis -FROM ${BASE_IMAGE} +FROM debian:bookworm-slim # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added RUN set -eux; \ From 6c299a6986fa7aabaf9683002dc6c528a6969dc3 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Fri, 13 Sep 2024 11:58:07 +0300 Subject: [PATCH 003/220] Removal of multistage build In order to avoid potential issues while Docker attempt to build our images. We will remove the multistage build from our Dockerfiles. --- alpine/Dockerfile | 21 +++++++++++---------- debian/Dockerfile | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 7ae463e80..74d07c4dc 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -1,12 +1,3 @@ -FROM alpine:3.20 AS sources -ENV REDIS_VERSION=8.0-m01 -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c -WORKDIR /files -RUN apk add tar patch wget -RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; echo "$REDIS_DOWNLOAD_SHA redis.tar.gz" | sha256sum -c -; -RUN tar -xvf redis.tar.gz && mv redis-$REDIS_VERSION redis - FROM alpine:3.20 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -52,7 +43,11 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw set -eux; \ +ENV REDIS_VERSION=8.0-m01 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c + +RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ coreutils \ @@ -90,6 +85,12 @@ RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw ;\ fi; \ \ + wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ + echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ + mkdir -p /usr/src/redis; \ + tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \ + rm redis.tar.gz; \ + \ # disable Redis protected mode [1] as it is unnecessary in context of Docker # (ports are not automatically exposed when running inside Docker, but rather explicitly by specifying -p / -P) # [1]: https://github.com/redis/redis/commit/edd4d555df57dc84265fdfb4ef59a4678832f6da diff --git a/debian/Dockerfile b/debian/Dockerfile index 20c3d7d79..c51a537d9 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -1,12 +1,3 @@ -FROM debian:bookworm-slim AS sources -ENV REDIS_VERSION=8.0-m01 -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c -WORKDIR /files -RUN apt-get update && apt-get install -y tar patch wget -RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; echo "$REDIS_DOWNLOAD_SHA redis.tar.gz" | sha256sum -c -; -RUN tar -xvf redis.tar.gz && mv redis-$REDIS_VERSION redis - FROM debian:bookworm-slim # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added @@ -59,7 +50,11 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw set -eux; \ +ENV REDIS_VERSION=8.0-m01 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c + +RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ @@ -98,6 +93,12 @@ RUN --mount=type=bind,from=sources,source=/files/redis,target=/usr/src/redis,rw \ rm -rf /var/lib/apt/lists/*; \ \ + wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ + echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ + mkdir -p /usr/src/redis; \ + tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \ + rm redis.tar.gz; \ + \ # disable Redis protected mode [1] as it is unnecessary in context of Docker # (ports are not automatically exposed when running inside Docker, but rather explicitly by specifying -p / -P) # [1]: https://github.com/redis/redis/commit/edd4d555df57dc84265fdfb4ef59a4678832f6da From f13e07666e456ef6b966651cc0890e4b73841ea5 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Fri, 13 Sep 2024 20:01:45 +0300 Subject: [PATCH 004/220] Use relative path for Dockerfile copies (#412) In order to align with requirements of the official docker library images' build process. https://github.com/docker-library/official-images/pull/17549#issuecomment-2349364178 --- .github/actions/build-and-tag-locally/action.yml | 6 ++---- alpine/Dockerfile | 3 ++- debian/Dockerfile | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 149661d77..66015a618 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -84,8 +84,7 @@ runs: - name: Build uses: docker/build-push-action@v6 with: - context: . - file: ${{ inputs.distribution }}/Dockerfile + context: ${{ inputs.distribution }} push: false load: true platforms: ${{ inputs.platform }} @@ -203,8 +202,7 @@ runs: uses: docker/build-push-action@v6 if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64","arm64"]'), steps.platform.outputs.display_name) }} with: - context: . - file: ${{ inputs.distribution }}/Dockerfile + context: ${{ inputs.distribution }} push: true tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 74d07c4dc..02967ddca 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -132,6 +132,7 @@ RUN set -eux; \ ; \ \ make -C /usr/src/redis distclean; \ + rm -r /usr/src/redis; \ \ runDeps="$( \ scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \ @@ -148,7 +149,7 @@ RUN mkdir /data && chown redis:redis /data VOLUME /data WORKDIR /data -COPY alpine/docker-entrypoint.sh /usr/local/bin/ +COPY docker-entrypoint.sh /usr/local/bin/ ENTRYPOINT ["docker-entrypoint.sh"] EXPOSE 6379 diff --git a/debian/Dockerfile b/debian/Dockerfile index c51a537d9..05644b0eb 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -139,6 +139,7 @@ RUN set -eux; \ ; \ \ make -C /usr/src/redis distclean; \ + rm -r /usr/src/redis; \ \ apt-mark auto '.*' > /dev/null; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ @@ -160,7 +161,7 @@ RUN mkdir /data && chown redis:redis /data VOLUME /data WORKDIR /data -COPY debian/docker-entrypoint.sh /usr/local/bin/ +COPY docker-entrypoint.sh /usr/local/bin/ ENTRYPOINT ["docker-entrypoint.sh"] EXPOSE 6379 From af8fe134a94d9d3ac4c696a5d8fd0096e7df6794 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Sat, 14 Sep 2024 10:14:07 +0300 Subject: [PATCH 005/220] Improve docker-entrypoint.sh scripts (#413) - Provide preliminary checks for the modules, with appropriate warning messages. This is to support entry command overrides. https://github.com/docker-library/official-images/pull/17549#issuecomment-2350726819 --- alpine/docker-entrypoint.sh | 37 +++++++++++++++++++++++++++++++------ debian/docker-entrypoint.sh | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index 77e274f96..114e094d5 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -21,14 +21,39 @@ if [ "$um" = '0022' ]; then umask 0077 fi -modules_dir="/usr/local/lib/redis/modules/" command="exec \"\$@\"" +if [ "$1" = 'redis-server' ]; then + echo "Starting Redis Server" + modules_dir="/usr/local/lib/redis/modules/" + + if [ ! -d "$modules_dir" ]; then + echo "Warning: Default Redis modules directory $modules_dir does not exist." + elif [ -n "$(ls -A $modules_dir 2>/dev/null)" ]; then + for module in "$modules_dir"/*.so; + do + if [ ! -s "$module" ]; then + echo "Skipping module $module: file has no size." + continue + fi + + if [ -d "$module" ]; then + echo "Skipping module $module: is a directory." + continue + fi + + if [ ! -r "$module" ]; then + echo "Skipping module $module: file is not readable." + continue + fi -if [ -n "$(ls -A $modules_dir)" ]; then - for module in "$modules_dir"/*.so; - do - command="$command --loadmodule $module" - done + if [ ! -x "$module" ]; then + echo "Warning: Module $module is not executable." + fi + + command="$command --loadmodule $module" + done + fi fi + eval "$command" \ No newline at end of file diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh index 77e274f96..114e094d5 100755 --- a/debian/docker-entrypoint.sh +++ b/debian/docker-entrypoint.sh @@ -21,14 +21,39 @@ if [ "$um" = '0022' ]; then umask 0077 fi -modules_dir="/usr/local/lib/redis/modules/" command="exec \"\$@\"" +if [ "$1" = 'redis-server' ]; then + echo "Starting Redis Server" + modules_dir="/usr/local/lib/redis/modules/" + + if [ ! -d "$modules_dir" ]; then + echo "Warning: Default Redis modules directory $modules_dir does not exist." + elif [ -n "$(ls -A $modules_dir 2>/dev/null)" ]; then + for module in "$modules_dir"/*.so; + do + if [ ! -s "$module" ]; then + echo "Skipping module $module: file has no size." + continue + fi + + if [ -d "$module" ]; then + echo "Skipping module $module: is a directory." + continue + fi + + if [ ! -r "$module" ]; then + echo "Skipping module $module: file is not readable." + continue + fi -if [ -n "$(ls -A $modules_dir)" ]; then - for module in "$modules_dir"/*.so; - do - command="$command --loadmodule $module" - done + if [ ! -x "$module" ]; then + echo "Warning: Module $module is not executable." + fi + + command="$command --loadmodule $module" + done + fi fi + eval "$command" \ No newline at end of file From 1b88507c82861395a5c1b354baab795c73c051e3 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Thu, 19 Sep 2024 16:07:41 +0300 Subject: [PATCH 006/220] Deprecate usage of eval in `docker-entrypoint.sh` (#415) `eval` is riskier due to potential security vulnerabilities and unpredictable behavior when handling arguments. --- alpine/docker-entrypoint.sh | 5 +++-- debian/docker-entrypoint.sh | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index 114e094d5..f7d408c02 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -48,12 +48,13 @@ if [ "$1" = 'redis-server' ]; then if [ ! -x "$module" ]; then echo "Warning: Module $module is not executable." + continue fi - command="$command --loadmodule $module" + set -- "$@" --loadmodule "$module" done fi fi -eval "$command" \ No newline at end of file +exec "$@" \ No newline at end of file diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh index 114e094d5..f7d408c02 100755 --- a/debian/docker-entrypoint.sh +++ b/debian/docker-entrypoint.sh @@ -48,12 +48,13 @@ if [ "$1" = 'redis-server' ]; then if [ ! -x "$module" ]; then echo "Warning: Module $module is not executable." + continue fi - command="$command --loadmodule $module" + set -- "$@" --loadmodule "$module" done fi fi -eval "$command" \ No newline at end of file +exec "$@" \ No newline at end of file From 3e867f9ac7c856ce5e40552b7cb71953ce51e7ed Mon Sep 17 00:00:00 2001 From: adamiBs Date: Mon, 23 Sep 2024 14:43:04 +0300 Subject: [PATCH 007/220] Remove leftover entry-point implementation (#417) --- alpine/docker-entrypoint.sh | 1 - debian/docker-entrypoint.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index f7d408c02..15e6c02c1 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -21,7 +21,6 @@ if [ "$um" = '0022' ]; then umask 0077 fi -command="exec \"\$@\"" if [ "$1" = 'redis-server' ]; then echo "Starting Redis Server" modules_dir="/usr/local/lib/redis/modules/" diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh index f7d408c02..15e6c02c1 100755 --- a/debian/docker-entrypoint.sh +++ b/debian/docker-entrypoint.sh @@ -21,7 +21,6 @@ if [ "$um" = '0022' ]; then umask 0077 fi -command="exec \"\$@\"" if [ "$1" = 'redis-server' ]; then echo "Starting Redis Server" modules_dir="/usr/local/lib/redis/modules/" From 6203acb65e690d21efee279615c876593b1eb1cf Mon Sep 17 00:00:00 2001 From: maxb-io <105273783+maxb-io@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:17:01 +0200 Subject: [PATCH 008/220] Add alpine support (#421) Co-authored-by: adamiBs --- .../actions/build-and-tag-locally/action.yml | 26 +++++--- .github/workflows/pre-merge.yml | 14 ++++- .gitignore | 1 - alpine/Dockerfile | 62 +++++++++++++------ debian/Dockerfile | 6 +- 5 files changed, 76 insertions(+), 33 deletions(-) delete mode 100644 .gitignore diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 66015a618..70c2a052f 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -45,6 +45,9 @@ runs: linux/arm/v5) plaform_name="arm-v5" ;; + linux/arm/v6) + plaform_name="arm-v6" + ;; linux/arm/v7) plaform_name="arm-v7" ;; @@ -57,6 +60,9 @@ runs: linux/ppc64le) plaform_name="ppc64le" ;; + linux/riscv64) + plaform_name="riscv64" + ;; linux/s390x) plaform_name="s390x" ;; @@ -100,31 +106,31 @@ runs: - name: Upload image uses: actions/upload-artifact@v4 with: - name: ${{ steps.platform.outputs.display_name }}-docker-image.tar + name: ${{ steps.platform.outputs.display_name }}-${{ inputs.distribution }}-docker-image.tar path: /tmp/image-${{ steps.platform.outputs.display_name }}.tar retention-days: 45 - name: Run container shell: bash - if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} run: | docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - name: Container Logs - if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker logs sanity-test-${{ steps.platform.outputs.display_name }} - name: Sanity Tests - if: ${{ contains(fromJSON('["amd64", "arm64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server - name: Verify installed modules - if: ${{ contains(fromJSON('["amd64", "arm64",]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} shell: bash run: | modules=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli module list) @@ -144,7 +150,7 @@ runs: fi - name: Test RedisBloom - if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.ADD popular_keys "redis:hash" @@ -154,7 +160,7 @@ runs: echo "RedisBloom test passed successfully" - name: Test RediSearch - if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli FT.CREATE redis_commands ON HASH PREFIX 1 cmd: SCHEMA name TEXT SORTABLE description TEXT @@ -169,7 +175,7 @@ runs: fi - name: Test RedisTimeSeries - if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.CREATE redis:cpu:usage RETENTION 86400 @@ -185,7 +191,7 @@ runs: fi - name: Test ReJSON - if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli JSON.SET redis:config $ '{"maxmemory":"2gb","maxmemory-policy":"allkeys-lru"}' @@ -200,7 +206,7 @@ runs: - name: Push image uses: docker/build-push-action@v6 - if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64","arm64"]'), steps.platform.outputs.display_name) }} + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} with: context: ${{ inputs.distribution }} push: true diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 53e3d0512..4eefba9d4 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -12,15 +12,27 @@ jobs: matrix: distribution: - debian - #- alpine + - alpine platform: - linux/amd64 - linux/i386 - linux/arm/v5 + - linux/arm/v6 - linux/arm/v7 - linux/mips64le - linux/ppc64le - linux/s390x + - linux/arm64 + - linux/riscv64 + exclude: + - distribution: alpine + platform: linux/mips64le + - distribution: alpine + platform: linux/arm/v5 + - distribution: debian + platform: linux/riscv64 + - distribution: debian + platform: linux/arm/v6 steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e43b0f988..000000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.DS_Store diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 02967ddca..60a3ee540 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -37,16 +37,15 @@ RUN set -eux; \ gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ gpgconf --kill all; \ - rm -f "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ apk del --no-network .gosu-fetch; \ chmod +x /usr/local/bin/gosu; \ gosu --version; \ gosu nobody true -ENV REDIS_VERSION=8.0-m01 -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c - +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m02.tar.gz +ENV REDIS_DOWNLOAD_SHA=c70d565c5403c5e8392942810e980b23478b82218f6069656ea51bc8978176c1 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ @@ -57,6 +56,7 @@ RUN set -eux; \ make \ musl-dev \ openssl-dev \ + g++ \ ; \ \ arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ @@ -67,26 +67,42 @@ RUN set -eux; \ esac; \ if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ apk add --no-cache --virtual .module-build-deps \ - git \ + autoconf \ + automake \ + bash \ + bsd-compat-headers \ + build-base \ + cargo \ + clang \ + clang18-libclang \ cmake \ curl \ - python3 \ + g++ \ + git \ + libffi-dev \ + libgcc \ + libtool \ + openssh \ + openssl \ + py-virtualenv \ + py3-cryptography \ py3-pip \ py3-virtualenv \ + python3 \ python3-dev \ - unzip \ rsync \ - clang \ - automake \ - autoconf \ - libtool \ - g++ \ - libgcc \ - ;\ + tar \ + unzip \ + which \ + xsimd \ + xz \ + ;\ fi; \ \ - wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ - echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ +# install required python packages for RedisJSON module + pip install -q --upgrade setuptools && pip install -q --upgrade pip && pip install -q addict toml jinja2 ramp-packer ;\ + wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ + echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ mkdir -p /usr/src/redis; \ tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \ rm redis.tar.gz; \ @@ -117,6 +133,11 @@ RUN set -eux; \ grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ \ export BUILD_TLS=yes; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + make -C /usr/src/redis/modules/redisjson get_source; \ + sed -i 's/^RUST_FLAGS=$/RUST_FLAGS += -C target-feature=-crt-static/' /usr/src/redis/modules/redisjson/src/Makefile ; \ + grep -E 'RUST_FLAGS' /usr/src/redis/modules/redisjson/src/Makefile; \ + fi; \ make -C /usr/src/redis -j "$(nproc)" all; \ make -C /usr/src/redis install; \ \ @@ -141,7 +162,12 @@ RUN set -eux; \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ apk add --no-network --virtual .redis-rundeps $runDeps; \ - apk del --no-network .build-deps; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + apk del --no-network .module-build-deps; \ + fi; \ + apk del --no-network .build-deps; \ + apk --purge del apk-tools ; \ + rm -fr ~/.cache/pip* rm -f /sbin/apk && rm -rf /etc/apk && rm -rf /lib/apk && rm -rf /usr/share/apk && rm -rf /var/lib/apk && rm -rf /usr/lib/python*; \ \ redis-cli --version; \ redis-server --version; diff --git a/debian/Dockerfile b/debian/Dockerfile index 05644b0eb..b9dc313cf 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -50,9 +50,8 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -ENV REDIS_VERSION=8.0-m01 -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=4c77c2218747505c50c43a45d12a067a3631a26d9397929da180e183b03e862c +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m02.tar.gz +ENV REDIS_DOWNLOAD_SHA=c70d565c5403c5e8392942810e980b23478b82218f6069656ea51bc8978176c1 RUN set -eux; \ \ @@ -63,6 +62,7 @@ RUN set -eux; \ wget \ dpkg-dev \ gcc \ + g++ \ libc6-dev \ libssl-dev \ make \ From f1e991818a8124502b5a4e8e6c7f4ae23d0c7bb4 Mon Sep 17 00:00:00 2001 From: maxb-io <105273783+maxb-io@users.noreply.github.com> Date: Wed, 30 Oct 2024 20:13:31 +0200 Subject: [PATCH 009/220] move PIP_BREAK_SYSTEM_PACKAGES=1 to RUN instead of setting it as anv var (#422) --- alpine/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 60a3ee540..e0e3e4172 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -45,7 +45,6 @@ RUN set -eux; \ ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m02.tar.gz ENV REDIS_DOWNLOAD_SHA=c70d565c5403c5e8392942810e980b23478b82218f6069656ea51bc8978176c1 -ENV PIP_BREAK_SYSTEM_PACKAGES=1 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ @@ -100,7 +99,7 @@ RUN set -eux; \ fi; \ \ # install required python packages for RedisJSON module - pip install -q --upgrade setuptools && pip install -q --upgrade pip && pip install -q addict toml jinja2 ramp-packer ;\ + pip install -q --upgrade setuptools && pip install -q --upgrade pip && PIP_BREAK_SYSTEM_PACKAGES=1 pip install -q addict toml jinja2 ramp-packer ;\ wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ mkdir -p /usr/src/redis; \ From 7109557d2a7612b292a6ff2712eba560dc5e70bc Mon Sep 17 00:00:00 2001 From: adamiBs Date: Tue, 21 Jan 2025 12:29:06 +0200 Subject: [PATCH 010/220] Redis CE 8.0 M03. (#430) --- .github/workflows/pre-merge.yml | 2 +- alpine/Dockerfile | 6 +++--- debian/Dockerfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 4eefba9d4..87585b26a 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -7,7 +7,7 @@ on: jobs: build-and-test: - runs-on: [ubuntu-latest] + runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} strategy: matrix: distribution: diff --git a/alpine/Dockerfile b/alpine/Dockerfile index e0e3e4172..657e9ba6b 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20 +FROM alpine:3.21 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added RUN set -eux; \ @@ -43,8 +43,8 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m02.tar.gz -ENV REDIS_DOWNLOAD_SHA=c70d565c5403c5e8392942810e980b23478b82218f6069656ea51bc8978176c1 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m03.tar.gz +ENV REDIS_DOWNLOAD_SHA=46d9a1dfb22c48e4a114ad9b620986d8bbca811d347b1fe759b5bb7750dac5ad RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index b9dc313cf..f30719761 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -50,8 +50,8 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m02.tar.gz -ENV REDIS_DOWNLOAD_SHA=c70d565c5403c5e8392942810e980b23478b82218f6069656ea51bc8978176c1 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m03.tar.gz +ENV REDIS_DOWNLOAD_SHA=46d9a1dfb22c48e4a114ad9b620986d8bbca811d347b1fe759b5bb7750dac5ad RUN set -eux; \ \ From 1d61ffb74b806f51c3691e296d827c3baacc5056 Mon Sep 17 00:00:00 2001 From: Alexander Dobrzhansky Date: Wed, 19 Mar 2025 18:21:03 +0100 Subject: [PATCH 011/220] Redis CE 8.0 Milestone 04. (#436) --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 657e9ba6b..73ef1034a 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -43,8 +43,8 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m03.tar.gz -ENV REDIS_DOWNLOAD_SHA=46d9a1dfb22c48e4a114ad9b620986d8bbca811d347b1fe759b5bb7750dac5ad +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz +ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index f30719761..69ee90fbd 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -50,8 +50,8 @@ RUN set -eux; \ gosu --version; \ gosu nobody true -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m03.tar.gz -ENV REDIS_DOWNLOAD_SHA=46d9a1dfb22c48e4a114ad9b620986d8bbca811d347b1fe759b5bb7750dac5ad +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz +ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 RUN set -eux; \ \ From 3b9471e7803ad7773bb2926ebc6535bec78ac3ce Mon Sep 17 00:00:00 2001 From: Peter-Sh Date: Fri, 21 Mar 2025 15:10:33 +0200 Subject: [PATCH 012/220] Use setpriv instead of gosu for dropping privileges in the entrypoint (#435) * Use setpriv instead of gosu to drop privileges Changes: setpriv is used instead of gosu with the following flags: * Set reuid and regid to redis user and group * Clear all supplementary groups * Set bouding capabilities to an empty list * Enable no-new-privs bit * Set securebit to exclude regaining capabilities --- alpine/Dockerfile | 32 ++------------------------------ alpine/docker-entrypoint.sh | 29 +++++++++++++++++++++++++---- debian/Dockerfile | 37 ------------------------------------- debian/docker-entrypoint.sh | 29 +++++++++++++++++++++++++---- 4 files changed, 52 insertions(+), 75 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 73ef1034a..61cca3761 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -11,38 +11,10 @@ RUN set -eux; \ apk add --no-cache \ # add tzdata for https://github.com/docker-library/redis/issues/138 tzdata \ +# we need setpriv package as busybox provides very limited functionality + setpriv \ ; -# grab gosu for easy step-down from root -# https://github.com/tianon/gosu/releases -ENV GOSU_VERSION=1.17 -RUN set -eux; \ - apk add --no-cache --virtual .gosu-fetch gnupg; \ - arch="$(apk --print-arch)"; \ - case "$arch" in \ - 'x86_64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-amd64'; sha256='bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3' ;; \ - 'aarch64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-arm64'; sha256='c3805a85d17f4454c23d7059bcb97e1ec1af272b90126e79ed002342de08389b' ;; \ - 'armhf') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ - 'x86') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-i386'; sha256='087dbb8fe479537e64f9c86fa49ff3b41dee1cbd28739a19aaef83dc8186b1ca' ;; \ - 'ppc64le') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-ppc64el'; sha256='1891acdcfa70046818ab6ed3c52b9d42fa10fbb7b340eb429c8c7849691dbd76' ;; \ - 'riscv64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-riscv64'; sha256='38a6444b57adce135c42d5a3689f616fc7803ddc7a07ff6f946f2ebc67a26ba6' ;; \ - 's390x') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-s390x'; sha256='69873bab588192f760547ca1f75b27cfcf106e9f7403fee6fd0600bc914979d0' ;; \ - 'armv7') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ - *) echo >&2 "error: unsupported gosu architecture: '$arch'"; exit 1 ;; \ - esac; \ - wget -O /usr/local/bin/gosu.asc "$url.asc"; \ - wget -O /usr/local/bin/gosu "$url"; \ - echo "$sha256 */usr/local/bin/gosu" | sha256sum -c -; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - apk del --no-network .gosu-fetch; \ - chmod +x /usr/local/bin/gosu; \ - gosu --version; \ - gosu nobody true - ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 RUN set -eux; \ diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index 15e6c02c1..ab5befbf8 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -1,16 +1,37 @@ #!/bin/sh set -e +has_cap() { + /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'$1'\b' +} + # first arg is `-f` or `--some-option` # or first arg is `something.conf` if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then set -- redis-server "$@" fi -# allow the container to be started with `--user` -if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then +CMD=$(realpath $(command -v "$1") 2>/dev/null || :) +# drop privileges only if our uid is 0 (container started without explicit --user) +# and we have capabilities required to drop privs +if has_cap setuid && has_cap setgid && \ + [ \( "$CMD" = '/usr/local/bin/redis-server' -o "$CMD" = '/usr/local/bin/redis-sentinel' \) -a "$(id -u)" = '0' ]; then find . \! -user redis -exec chown redis '{}' + - exec gosu redis "$0" "$@" + CAPS_TO_KEEP="" + if has_cap sys_resource; then + # we have sys_resource capability, keep it available for redis + # as redis may use it to increase open files limit + CAPS_TO_KEEP=",+sys_resource" + fi + exec /usr/bin/setpriv \ + --reuid redis \ + --regid redis \ + --clear-groups \ + --nnp \ + --inh-caps=-all$CAPS_TO_KEEP \ + --ambient-caps=-all$CAPS_TO_KEEP \ + --bounding-set=-all$CAPS_TO_KEEP \ + "$0" "$@" fi # set an appropriate umask (if one isn't set already) @@ -56,4 +77,4 @@ if [ "$1" = 'redis-server' ]; then fi -exec "$@" \ No newline at end of file +exec "$@" diff --git a/debian/Dockerfile b/debian/Dockerfile index 69ee90fbd..8a6f78900 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,45 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -# grab gosu for easy step-down from root -# https://github.com/tianon/gosu/releases -ENV GOSU_VERSION=1.17 -RUN set -eux; \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates gnupg wget; \ - rm -rf /var/lib/apt/lists/*; \ - arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - case "$arch" in \ - 'amd64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-amd64'; sha256='bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3' ;; \ - 'arm64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-arm64'; sha256='c3805a85d17f4454c23d7059bcb97e1ec1af272b90126e79ed002342de08389b' ;; \ - 'armel') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armel'; sha256='f9969910fa141140438c998cfa02f603bf213b11afd466dcde8fa940e700945d' ;; \ - 'i386') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-i386'; sha256='087dbb8fe479537e64f9c86fa49ff3b41dee1cbd28739a19aaef83dc8186b1ca' ;; \ - 'mips64el') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-mips64el'; sha256='87140029d792595e660be0015341dfa1c02d1181459ae40df9f093e471d75b70' ;; \ - 'ppc64el') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-ppc64el'; sha256='1891acdcfa70046818ab6ed3c52b9d42fa10fbb7b340eb429c8c7849691dbd76' ;; \ - 'riscv64') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-riscv64'; sha256='38a6444b57adce135c42d5a3689f616fc7803ddc7a07ff6f946f2ebc67a26ba6' ;; \ - 's390x') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-s390x'; sha256='69873bab588192f760547ca1f75b27cfcf106e9f7403fee6fd0600bc914979d0' ;; \ - 'armhf') url='https://github.com/tianon/gosu/releases/download/1.17/gosu-armhf'; sha256='e5866286277ff2a2159fb9196fea13e0a59d3f1091ea46ddb985160b94b6841b' ;; \ - *) echo >&2 "error: unsupported gosu architecture: '$arch'"; exit 1 ;; \ - esac; \ - wget -O /usr/local/bin/gosu.asc "$url.asc"; \ - wget -O /usr/local/bin/gosu "$url"; \ - echo "$sha256 */usr/local/bin/gosu" | sha256sum -c -; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - chmod +x /usr/local/bin/gosu; \ - gosu --version; \ - gosu nobody true - ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 - RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh index 15e6c02c1..ab5befbf8 100755 --- a/debian/docker-entrypoint.sh +++ b/debian/docker-entrypoint.sh @@ -1,16 +1,37 @@ #!/bin/sh set -e +has_cap() { + /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'$1'\b' +} + # first arg is `-f` or `--some-option` # or first arg is `something.conf` if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then set -- redis-server "$@" fi -# allow the container to be started with `--user` -if [ "$1" = 'redis-server' -a "$(id -u)" = '0' ]; then +CMD=$(realpath $(command -v "$1") 2>/dev/null || :) +# drop privileges only if our uid is 0 (container started without explicit --user) +# and we have capabilities required to drop privs +if has_cap setuid && has_cap setgid && \ + [ \( "$CMD" = '/usr/local/bin/redis-server' -o "$CMD" = '/usr/local/bin/redis-sentinel' \) -a "$(id -u)" = '0' ]; then find . \! -user redis -exec chown redis '{}' + - exec gosu redis "$0" "$@" + CAPS_TO_KEEP="" + if has_cap sys_resource; then + # we have sys_resource capability, keep it available for redis + # as redis may use it to increase open files limit + CAPS_TO_KEEP=",+sys_resource" + fi + exec /usr/bin/setpriv \ + --reuid redis \ + --regid redis \ + --clear-groups \ + --nnp \ + --inh-caps=-all$CAPS_TO_KEEP \ + --ambient-caps=-all$CAPS_TO_KEEP \ + --bounding-set=-all$CAPS_TO_KEEP \ + "$0" "$@" fi # set an appropriate umask (if one isn't set already) @@ -56,4 +77,4 @@ if [ "$1" = 'redis-server' ]; then fi -exec "$@" \ No newline at end of file +exec "$@" From f3cfc256e913880e5d5eefc794e220c6b0733f22 Mon Sep 17 00:00:00 2001 From: Alexander Dobrzhansky Date: Mon, 7 Apr 2025 14:10:56 +0200 Subject: [PATCH 013/220] Redis CE 8.0 RC1 (#437) --- alpine/Dockerfile | 99 +++++++++++++++++++++++------------------------ debian/Dockerfile | 9 ++--- 2 files changed, 52 insertions(+), 56 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 61cca3761..d7f543166 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,9 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; - -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz -ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-rc1.tar.gz +ENV REDIS_DOWNLOAD_SHA=3f8283dcbaf3f8297607c2595ccd9b2b9785a0e88f4007c882dd60846ffec28c RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ @@ -27,9 +26,8 @@ RUN set -eux; \ make \ musl-dev \ openssl-dev \ - g++ \ - ; \ - \ + g++; \ + \ arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ case "$arch" in \ 'amd64') export BUILD_WITH_MODULES=yes; export INSTALL_RUST_TOOLCHAIN=yes; export DISABLE_WERRORS=yes ;; \ @@ -37,43 +35,42 @@ RUN set -eux; \ *) echo >&2 "Modules are NOT supported! unsupported architecture: '$arch'"; export BUILD_WITH_MODULES=no ;; \ esac; \ if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ - apk add --no-cache --virtual .module-build-deps \ - autoconf \ - automake \ - bash \ - bsd-compat-headers \ - build-base \ - cargo \ - clang \ - clang18-libclang \ - cmake \ - curl \ - g++ \ - git \ - libffi-dev \ - libgcc \ - libtool \ - openssh \ - openssl \ - py-virtualenv \ - py3-cryptography \ - py3-pip \ - py3-virtualenv \ - python3 \ - python3-dev \ - rsync \ - tar \ - unzip \ - which \ - xsimd \ - xz \ - ;\ + apk add --no-cache --virtual .module-build-deps \ + autoconf \ + automake \ + bash \ + bsd-compat-headers \ + build-base \ + cargo \ + clang \ + clang18-libclang \ + cmake \ + curl \ + g++ \ + git \ + libffi-dev \ + libgcc \ + libtool \ + openssh \ + openssl \ + py-virtualenv \ + py3-cryptography \ + py3-pip \ + py3-virtualenv \ + python3 \ + python3-dev \ + rsync \ + tar \ + unzip \ + which \ + xsimd \ + xz; \ fi; \ \ -# install required python packages for RedisJSON module +# install required python packages for RedisJSON module pip install -q --upgrade setuptools && pip install -q --upgrade pip && PIP_BREAK_SYSTEM_PACKAGES=1 pip install -q addict toml jinja2 ramp-packer ;\ - wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ - echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ + wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \ + echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \ mkdir -p /usr/src/redis; \ tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \ rm redis.tar.gz; \ @@ -104,11 +101,11 @@ RUN set -eux; \ grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ \ export BUILD_TLS=yes; \ - if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ - make -C /usr/src/redis/modules/redisjson get_source; \ - sed -i 's/^RUST_FLAGS=$/RUST_FLAGS += -C target-feature=-crt-static/' /usr/src/redis/modules/redisjson/src/Makefile ; \ - grep -E 'RUST_FLAGS' /usr/src/redis/modules/redisjson/src/Makefile; \ - fi; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + make -C /usr/src/redis/modules/redisjson get_source; \ + sed -i 's/^RUST_FLAGS=$/RUST_FLAGS += -C target-feature=-crt-static/' /usr/src/redis/modules/redisjson/src/Makefile ; \ + grep -E 'RUST_FLAGS' /usr/src/redis/modules/redisjson/src/Makefile; \ + fi; \ make -C /usr/src/redis -j "$(nproc)" all; \ make -C /usr/src/redis install; \ \ @@ -133,12 +130,12 @@ RUN set -eux; \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ apk add --no-network --virtual .redis-rundeps $runDeps; \ - if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ - apk del --no-network .module-build-deps; \ - fi; \ - apk del --no-network .build-deps; \ - apk --purge del apk-tools ; \ - rm -fr ~/.cache/pip* rm -f /sbin/apk && rm -rf /etc/apk && rm -rf /lib/apk && rm -rf /usr/share/apk && rm -rf /var/lib/apk && rm -rf /usr/lib/python*; \ + if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ + apk del --no-network .module-build-deps; \ + fi; \ + apk del --no-network .build-deps; \ + apk --purge del apk-tools ; \ + rm -fr ~/.cache/pip* rm -f /sbin/apk && rm -rf /etc/apk && rm -rf /lib/apk && rm -rf /usr/share/apk && rm -rf /var/lib/apk && rm -rf /usr/lib/python*; \ \ redis-cli --version; \ redis-server --version; diff --git a/debian/Dockerfile b/debian/Dockerfile index 8a6f78900..5ee068d92 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-m04.tar.gz -ENV REDIS_DOWNLOAD_SHA=6902a938c629a33f14d49881b1b60e6621c29e445554f882ce7ec48f2743d516 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-rc1.tar.gz +ENV REDIS_DOWNLOAD_SHA=3f8283dcbaf3f8297607c2595ccd9b2b9785a0e88f4007c882dd60846ffec28c RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ @@ -25,11 +25,10 @@ RUN set -eux; \ wget \ dpkg-dev \ gcc \ - g++ \ + g++ \ libc6-dev \ libssl-dev \ - make \ - ; \ + make; \ \ arch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ case "$arch" in \ From 7fc7e5625cd84b832db85561cb73b1bef78583fa Mon Sep 17 00:00:00 2001 From: adamiBs Date: Sun, 4 May 2025 10:19:33 +0300 Subject: [PATCH 014/220] Redis 8.0.0 (#443) Redis 8.0: http://redis.io/blog/redis-8-ga/ --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index d7f543166..4c2755cba 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-rc1.tar.gz -ENV REDIS_DOWNLOAD_SHA=3f8283dcbaf3f8297607c2595ccd9b2b9785a0e88f4007c882dd60846ffec28c +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=6d1b428d289426b68cff933d61f2d5c0a44a316f17236c51fbb33bc9e5c5a385 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 5ee068d92..469bde1a7 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0-rc1.tar.gz -ENV REDIS_DOWNLOAD_SHA=3f8283dcbaf3f8297607c2595ccd9b2b9785a0e88f4007c882dd60846ffec28c +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=6d1b428d289426b68cff933d61f2d5c0a44a316f17236c51fbb33bc9e5c5a385 RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 65a9c9078e2b49f038356e76bc247a4baa976e03 Mon Sep 17 00:00:00 2001 From: Peter-Sh Date: Tue, 13 May 2025 18:58:05 +0300 Subject: [PATCH 015/220] Don't remove apk tools and related files (#448) --- alpine/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 4c2755cba..26a39d9ac 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -134,8 +134,7 @@ RUN set -eux; \ apk del --no-network .module-build-deps; \ fi; \ apk del --no-network .build-deps; \ - apk --purge del apk-tools ; \ - rm -fr ~/.cache/pip* rm -f /sbin/apk && rm -rf /etc/apk && rm -rf /lib/apk && rm -rf /usr/share/apk && rm -rf /var/lib/apk && rm -rf /usr/lib/python*; \ + rm -rf ~/.cache ~/.gitconfig; \ \ redis-cli --version; \ redis-server --version; From fe864e383ae7c5c891a694f9e0c16f8459a62234 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Tue, 13 May 2025 19:54:20 +0300 Subject: [PATCH 016/220] Redis 8.0.1. (#449) --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 26a39d9ac..d6d131576 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=6d1b428d289426b68cff933d61f2d5c0a44a316f17236c51fbb33bc9e5c5a385 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=5e347d3532ff15bb888a78d851e87cf5cc1956edd32b5d4a0cac3220da0a5a0b RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 469bde1a7..fc46b649e 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=6d1b428d289426b68cff933d61f2d5c0a44a316f17236c51fbb33bc9e5c5a385 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=5e347d3532ff15bb888a78d851e87cf5cc1956edd32b5d4a0cac3220da0a5a0b RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From d6d0c3119315d10cc78db6fd086460706bc9566d Mon Sep 17 00:00:00 2001 From: Peter-Sh Date: Thu, 29 May 2025 17:01:05 +0300 Subject: [PATCH 017/220] Fix permissions for config files and introduce entrypoint testing (#451) This PR attempts to address permissions and ownership issues in the Redis config and data directories, but only for simple, default cases where it's safe to assume we won't overwrite or alter user-specific files (e.g., if a user's home directory is mistakenly mounted). **Key Changes:** * Fixes config file and directory permissions when they are insufficient for server startup. * Introduces the `SKIP_FIX_PERMS` environment variable to completely skip permission fixes, if desired. * Introduces the `SKIP_DROP_PRIVS` environment variable to optionally disable privilege dropping. This is not recommended, but may be necessary for compatibility with older image versions. * Adds a comprehensive entrypoint test suite that simulates a wide range of real-world scenarios. **Breaking Change Notice:** Users who previously relied on automatic permission fixes in the data directory but have non-standard configurations (e.g., a custom `appendonlydir`) or unrelated files in the data volume may find that these fixes no longer apply. We've chosen to err on the side of caution to avoid unintended data loss or misconfiguration caused by overly aggressive permission handling. Fixes: https://github.com/redis/docker-library-redis/issues/446 --- .../actions/build-and-tag-locally/action.yml | 22 +- alpine/docker-entrypoint.sh | 128 +- debian/docker-entrypoint.sh | 130 +- test/run-entrypoint-tests.sh | 649 +++++++ test/shunit2 | 1612 +++++++++++++++++ 5 files changed, 2515 insertions(+), 26 deletions(-) create mode 100755 test/run-entrypoint-tests.sh create mode 100755 test/shunit2 diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 70c2a052f..f67023fd5 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -204,6 +204,26 @@ runs: exit 1 fi + - name: Test the entrypoint + id: test_entrypoint + if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} + shell: bash + run: > + cd test && env + PLATFORM=${{ steps.platform.outputs.display_name }} + REDIS_IMG=${{ github.sha }}:${{ steps.platform.outputs.display_name }} + ./run-entrypoint-tests.sh + -- --output-junit-xml=report-entrypoint.xml + + - name: Test Report + uses: dorny/test-reporter@v2 + # run this step even if previous step failed, but not if it was skipped + if: ${{ !cancelled() && steps.test_entrypoint.conclusion != 'skipped' }} + with: + name: Entrypoint Tests + path: test/report-entrypoint.xml + reporter: java-junit + - name: Push image uses: docker/build-push-action@v6 if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} @@ -212,4 +232,4 @@ runs: push: true tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file + cache-to: type=gha,mode=max diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index ab5befbf8..476913d93 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -1,8 +1,93 @@ #!/bin/sh set -e +SETPRIV="/usr/bin/setpriv --reuid redis --regid redis --clear-groups" +IS_REDIS_SENTINEL="" +IS_REDIS_SERVER="" +CONFIG="" + +SKIP_FIX_PERMS_NOTICE="Use SKIP_FIX_PERMS=1 to skip permission changes." + +# functions has_cap() { - /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'$1'\b' + /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'"$1"'\b' +} + +check_for_sentinel() { + CMD="$1" + shift + if [ "$CMD" = '/usr/local/bin/redis-server' ]; then + for arg in "$@"; do + if [ "$arg" = "--sentinel" ]; then + return 0 + fi + done + fi + + if [ "$CMD" = '/usr/local/bin/redis-sentinel' ]; then + return 0 + fi + + return 1 +} + +# Note: Change permissions only in simple, default cases to avoid affecting +# unexpected or user-specific files. + +fix_data_dir_perms() { + # Expecting only *.rdb files and default appendonlydir; skip if others are found. + unknown_file="$(find . -mindepth 1 -maxdepth 1 \ + -not \( -name \*.rdb -or \( -type d -and -name appendonlydir \) \) \ + -print -quit)" + if [ -z "$unknown_file" ]; then + find . -print0 | fix_perms_and_owner rw + else + echo "Notice: Unknown file '$unknown_file' found in data dir. Permissions will not be modified. $SKIP_FIX_PERMS_NOTICE" + fi +} + +fix_config_perms() { + config="$1" + mode="$2" + + if [ ! -f "$config" ]; then + return 0 + fi + + confdir="$(dirname "$config")" + if [ ! -d "$confdir" ]; then + return 0 + fi + + # Expecting only the config file; skip if others are found. + pattern=$(printf "%s" "$(basename "$config")" | sed 's/[][?*]/\\&/g') + unknown_file=$(find "$confdir" -mindepth 1 -maxdepth 1 -not -name "$pattern" -print -quit) + + if [ -z "$unknown_file" ]; then + printf '%s\0%s\0' "$confdir" "$config" | fix_perms_and_owner "$mode" + else + echo "Notice: Unknown file '$unknown_file' found in '$confdir'. Permissions will not be modified. $SKIP_FIX_PERMS_NOTICE" + + fi +} + +fix_perms_and_owner() { + mode="$1" + + # shellcheck disable=SC3045 + while IFS= read -r -d '' file; do + if [ "$mode" = "rw" ] && $SETPRIV test -r "$file" -a -w "$file"; then + continue + elif [ "$mode" = "r" ] && $SETPRIV test -r "$file"; then + continue + fi + new_mode=$mode + if [ -d "$file" ]; then + new_mode=${mode}x + fi + err=$(chown redis "$file" 2>&1) || echo "Warning: cannot change owner to 'redis' for '$file': $err. $SKIP_FIX_PERMS_NOTICE" + err=$(chmod "u+$new_mode" "$file" 2>&1) || echo "Warning: cannot change mode to 'u+$new_mode' for '$file': $err. $SKIP_FIX_PERMS_NOTICE" + done } # first arg is `-f` or `--some-option` @@ -10,23 +95,43 @@ has_cap() { if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then set -- redis-server "$@" fi +CMD=$(command -v "$1" 2>/dev/null || :) -CMD=$(realpath $(command -v "$1") 2>/dev/null || :) -# drop privileges only if our uid is 0 (container started without explicit --user) +if [ "$(readlink -f "$CMD")" = '/usr/local/bin/redis-server' ]; then + IS_REDIS_SERVER=1 +fi + +if check_for_sentinel "$CMD" "$@"; then + IS_REDIS_SENTINEL=1 +fi + +# if is server and its first arg is not an option then it's a config +if [ "$IS_REDIS_SERVER" ] && [ "${2#-}" = "$2" ]; then + CONFIG="$2" +fi + +# drop privileges only if +# we are starting either server or sentinel +# our uid is 0 (container started without explicit --user) # and we have capabilities required to drop privs -if has_cap setuid && has_cap setgid && \ - [ \( "$CMD" = '/usr/local/bin/redis-server' -o "$CMD" = '/usr/local/bin/redis-sentinel' \) -a "$(id -u)" = '0' ]; then - find . \! -user redis -exec chown redis '{}' + +if [ "$IS_REDIS_SERVER" ] && [ -z "$SKIP_DROP_PRIVS" ] && [ "$(id -u)" = '0' ] && has_cap setuid && has_cap setgid; then + if [ -z "$SKIP_FIX_PERMS" ]; then + # fix permissions + if [ "$IS_REDIS_SENTINEL" ]; then + fix_config_perms "$CONFIG" rw + else + fix_data_dir_perms + fix_config_perms "$CONFIG" r + fi + fi + CAPS_TO_KEEP="" if has_cap sys_resource; then # we have sys_resource capability, keep it available for redis # as redis may use it to increase open files limit CAPS_TO_KEEP=",+sys_resource" fi - exec /usr/bin/setpriv \ - --reuid redis \ - --regid redis \ - --clear-groups \ + exec $SETPRIV \ --nnp \ --inh-caps=-all$CAPS_TO_KEEP \ --ambient-caps=-all$CAPS_TO_KEEP \ @@ -42,7 +147,7 @@ if [ "$um" = '0022' ]; then umask 0077 fi -if [ "$1" = 'redis-server' ]; then +if [ "$IS_REDIS_SERVER" ] && ! [ "$IS_REDIS_SENTINEL" ]; then echo "Starting Redis Server" modules_dir="/usr/local/lib/redis/modules/" @@ -76,5 +181,4 @@ if [ "$1" = 'redis-server' ]; then fi fi - exec "$@" diff --git a/debian/docker-entrypoint.sh b/debian/docker-entrypoint.sh index ab5befbf8..d0a21fe4d 100755 --- a/debian/docker-entrypoint.sh +++ b/debian/docker-entrypoint.sh @@ -1,8 +1,93 @@ -#!/bin/sh +#!/bin/bash set -e +SETPRIV="/usr/bin/setpriv --reuid redis --regid redis --clear-groups" +IS_REDIS_SENTINEL="" +IS_REDIS_SERVER="" +CONFIG="" + +SKIP_FIX_PERMS_NOTICE="Use SKIP_FIX_PERMS=1 to skip permission changes." + +# functions has_cap() { - /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'$1'\b' + /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'"$1"'\b' +} + +check_for_sentinel() { + CMD="$1" + shift + if [ "$CMD" = '/usr/local/bin/redis-server' ]; then + for arg in "$@"; do + if [ "$arg" = "--sentinel" ]; then + return 0 + fi + done + fi + + if [ "$CMD" = '/usr/local/bin/redis-sentinel' ]; then + return 0 + fi + + return 1 +} + +# Note: Change permissions only in simple, default cases to avoid affecting +# unexpected or user-specific files. + +fix_data_dir_perms() { + # Expecting only *.rdb files and default appendonlydir; skip if others are found. + unknown_file="$(find . -mindepth 1 -maxdepth 1 \ + -not \( -name \*.rdb -or \( -type d -and -name appendonlydir \) \) \ + -print -quit)" + if [ -z "$unknown_file" ]; then + find . -print0 | fix_perms_and_owner rw + else + echo "Notice: Unknown file '$unknown_file' found in data dir. Permissions will not be modified. $SKIP_FIX_PERMS_NOTICE" + fi +} + +fix_config_perms() { + config="$1" + mode="$2" + + if [ ! -f "$config" ]; then + return 0 + fi + + confdir="$(dirname "$config")" + if [ ! -d "$confdir" ]; then + return 0 + fi + + # Expecting only the config file; skip if others are found. + pattern=$(printf "%s" "$(basename "$config")" | sed 's/[][?*]/\\&/g') + unknown_file=$(find "$confdir" -mindepth 1 -maxdepth 1 -not -name "$pattern" -print -quit) + + if [ -z "$unknown_file" ]; then + printf '%s\0%s\0' "$confdir" "$config" | fix_perms_and_owner "$mode" + else + echo "Notice: Unknown file '$unknown_file' found in '$confdir'. Permissions will not be modified. $SKIP_FIX_PERMS_NOTICE" + + fi +} + +fix_perms_and_owner() { + mode="$1" + + # shellcheck disable=SC3045 + while IFS= read -r -d '' file; do + if [ "$mode" = "rw" ] && $SETPRIV test -r "$file" -a -w "$file"; then + continue + elif [ "$mode" = "r" ] && $SETPRIV test -r "$file"; then + continue + fi + new_mode=$mode + if [ -d "$file" ]; then + new_mode=${mode}x + fi + err=$(chown redis "$file" 2>&1) || echo "Warning: cannot change owner to 'redis' for '$file': $err. $SKIP_FIX_PERMS_NOTICE" + err=$(chmod "u+$new_mode" "$file" 2>&1) || echo "Warning: cannot change mode to 'u+$new_mode' for '$file': $err. $SKIP_FIX_PERMS_NOTICE" + done } # first arg is `-f` or `--some-option` @@ -10,23 +95,43 @@ has_cap() { if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then set -- redis-server "$@" fi +CMD=$(command -v "$1" 2>/dev/null || :) -CMD=$(realpath $(command -v "$1") 2>/dev/null || :) -# drop privileges only if our uid is 0 (container started without explicit --user) +if [ "$(readlink -f "$CMD")" = '/usr/local/bin/redis-server' ]; then + IS_REDIS_SERVER=1 +fi + +if check_for_sentinel "$CMD" "$@"; then + IS_REDIS_SENTINEL=1 +fi + +# if is server and its first arg is not an option then it's a config +if [ "$IS_REDIS_SERVER" ] && [ "${2#-}" = "$2" ]; then + CONFIG="$2" +fi + +# drop privileges only if +# we are starting either server or sentinel +# our uid is 0 (container started without explicit --user) # and we have capabilities required to drop privs -if has_cap setuid && has_cap setgid && \ - [ \( "$CMD" = '/usr/local/bin/redis-server' -o "$CMD" = '/usr/local/bin/redis-sentinel' \) -a "$(id -u)" = '0' ]; then - find . \! -user redis -exec chown redis '{}' + +if [ "$IS_REDIS_SERVER" ] && [ -z "$SKIP_DROP_PRIVS" ] && [ "$(id -u)" = '0' ] && has_cap setuid && has_cap setgid; then + if [ -z "$SKIP_FIX_PERMS" ]; then + # fix permissions + if [ "$IS_REDIS_SENTINEL" ]; then + fix_config_perms "$CONFIG" rw + else + fix_data_dir_perms + fix_config_perms "$CONFIG" r + fi + fi + CAPS_TO_KEEP="" if has_cap sys_resource; then # we have sys_resource capability, keep it available for redis # as redis may use it to increase open files limit CAPS_TO_KEEP=",+sys_resource" fi - exec /usr/bin/setpriv \ - --reuid redis \ - --regid redis \ - --clear-groups \ + exec $SETPRIV \ --nnp \ --inh-caps=-all$CAPS_TO_KEEP \ --ambient-caps=-all$CAPS_TO_KEEP \ @@ -42,7 +147,7 @@ if [ "$um" = '0022' ]; then umask 0077 fi -if [ "$1" = 'redis-server' ]; then +if [ "$IS_REDIS_SERVER" ] && ! [ "$IS_REDIS_SENTINEL" ]; then echo "Starting Redis Server" modules_dir="/usr/local/lib/redis/modules/" @@ -76,5 +181,4 @@ if [ "$1" = 'redis-server' ]; then fi fi - exec "$@" diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh new file mode 100755 index 000000000..500fc3cec --- /dev/null +++ b/test/run-entrypoint-tests.sh @@ -0,0 +1,649 @@ +#!/bin/bash + +## +# +# These tests are designed to verify the correctness of the entrypoint behavior +# under different preconditions and arguments. As such, in some tests, it is +# expected that the Redis process may fail with errors. +# +# To run specific test use: +# +# REDIS_IMG=image ./test.sh -- specific_test_name +# +# To get verbose output use TEST_VERBOSE=1: +# +# TEST_VERBOSE=1 REDIS_IMG=image ./test.sh +# +# Uses shunit2: https://github.com/kward/shunit2 +# +# Requires sudo +# +## + +if [ -z "$REDIS_IMG" ]; then + echo "REDIS_IMG may not be empty" + exit 1 +fi +# By default create files owned by root to avoid intersecting with container user +HOST_UID=0 +HOST_GID=0 +if docker info 2>/dev/null | grep -qi rootless; then + # For rootless docker we have to use current user + HOST_UID=$(id -u) + HOST_GID=$(id -g) +fi +HOST_OWNER=$HOST_UID:$HOST_GID + +get_container_user_uid_gid_on_the_host() { + container_user="$1" + dir=$(mktemp -d -p .) + docker run --rm -v "$(pwd)/$dir":/w -w /w --entrypoint=/bin/sh "$REDIS_IMG" -c "chown $container_user ." + stat -c "%u %g" "$dir" + sudo rm -rf "$dir" +} + +# Detect how redis user and group from the container are mapped to the host ones +read -r REDIS_UID _ <<< "$(get_container_user_uid_gid_on_the_host redis:redis)" + +if [ "$REDIS_UID" == "$HOST_UID" ]; then + echo "Cannot test ownership as redis user uid is the same as current user" + exit 1 +fi + +# Helper functions # + +# creates one entry of directory structure +# used in combination with iterate_dir_structure_with +create_entry() { + dir="$1" + if [ "$type" = dir ]; then + sudo mkdir -p "$dir/$entry" + elif [ "$type" = file ]; then + sudo touch "$dir/$entry" + else + echo "Unknown type '$type' for entry '$entry'" + return 1 + fi + sudo chmod "$initial_mode" "$dir/$entry" + sudo chown "$initial_owner" "$dir/$entry" +} + +# asserts ownership and permissions for one entry from directory structure +# used in combination with iterate_dir_structure_with +assert_entry() { + dir="$1" + msg="$2" + actual_uid=$(sudo stat -c %u "$dir/$entry") + actual_mode=0$(sudo stat -c '%a' "$dir/$entry") + actual_mask=$(printf "0%03o" $(( actual_mode & expected_mode_mask ))) + assertEquals "$msg: Owner for $type '$entry'" "$expected_owner" "$actual_uid" + assertEquals "$msg: Mode mask for $type '$entry'" "$expected_mode_mask" "$actual_mask" +} + +# Iterates over directory structure assigning variables and executing command +# from the arguments for each entry. +# +# Directory structure is the following form: +# entry type initial owner -> expected uid initial mode -> expected mode mask +# . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 +# appendonlydir dir $HOST_OWNER -> $REDIS_UID 0333 -> 0600 +# dump.rdb file $HOST_OWNER -> $REDIS_UID 0333 -> 0600 +iterate_dir_structure_with() { + awk 'NF {print $1,$2,$3,$5,$6,$8}' \ + | while read -r \ + entry \ + type \ + initial_owner \ + expected_owner \ + initial_mode \ + expected_mode_mask; \ + do + "$@" + done +} + +# Ownership and permissions test helper. +# +# This function tests the entrypoint. +# +# The idea is to test data and config files ownerhsip and permissions before and after container has started. +# +# The function creates temporary directory and uses --dir-structure (see iterate_dir_structure_with and create_entry) +# to create files and directories in this temporary dir. +# +# The temporary dir is mounted into the --mount-target inside the container. +# +# Container is started using REDIS_IMG and the function arguments as CMD. +# +# After container exits, all file permissions and ownership are checked using expected values from --dir-structure (see assert_entry) +# +# Additionally --extra-assert function is invoked if present. +# +# Arguments: +# < --mount-target DIR > +# [ --dir-structure STRING ] +# [ --extra-assert FUNCTION ] +# [ --docker-flags FLAGS ] +# +# Positional arguments: +# $docker_cmd +run_docker_and_test_ownership() { + docker_flags= + extra_assert= + dir_structure= + while [[ $# -gt 0 ]]; do + case "$1" in + --dir-structure) + dir_structure="$2" + shift 2 + ;; + --mount-target) + mount_target="$2" + shift 2 + ;; + --docker-flags) + docker_flags="$2" + shift 2 + ;; + --extra-assert) + extra_assert="$2" + shift 2 + ;; + --*) + break + ;; + *) + break + ;; + esac + done + docker_cmd="$*" + + if [ -z "$mount_target" ]; then + fail "Mount target is empty" + return 1 + fi + + dir=$(mktemp -d -p .) + + iterate_dir_structure_with create_entry "$dir" <<<"$dir_structure" + + docker_run="docker run --rm -v "$(pwd)/$dir":$mount_target $docker_flags $REDIS_IMG $docker_cmd" + if [ "$TEST_VERBOSE" ]; then + echo -e "\n#### ownership test: $docker_cmd" + echo "running $docker_run" + echo "Before:" + sudo find "$dir" -exec ls -ald {} \+ + fi + + docker_output=$($docker_run 2>&1) + + if [ "$TEST_VERBOSE" ]; then + echo "After:" + sudo find "$dir" -exec ls -ald {} \+ + echo "Docker output:" + echo "$docker_output" + fi + + iterate_dir_structure_with assert_entry "$dir" "$docker_cmd" <<<"$dir_structure" + + if [ "$extra_assert" ]; then + $extra_assert + fi + + sudo rm -rf "$dir" +} + +# running redis-server using different forms +# -v option will make redis-server to either return version or fail (if config has been provided) +# either one is OK for us +run_docker_and_test_ownership_with_common_flags_for_server() { + run_docker_and_test_ownership "${common_flags[@]}" "$@" -v + run_docker_and_test_ownership "${common_flags[@]}" redis-server "$@" -v + run_docker_and_test_ownership "${common_flags[@]}" /usr/local/bin/redis-server "$@" -v +} + +# running redis-sentinel using different forms and --dumb-option +# expecting sentinel to fail, it's ok as we are only interested in entrypoint testing here +run_docker_and_test_ownership_with_common_flags_for_sentinel() { + run_docker_and_test_ownership "${common_flags[@]}" "$@" --sentinel --dumb-option + run_docker_and_test_ownership "${common_flags[@]}" redis-sentinel "$@" --dumb-option + run_docker_and_test_ownership "${common_flags[@]}" /usr/local/bin/redis-sentinel "$@" --dumb-option + run_docker_and_test_ownership "${common_flags[@]}" redis-server "$@" --sentinel --dumb-option + run_docker_and_test_ownership "${common_flags[@]}" /usr/local/bin/redis-server "$@" --sentinel --dumb-option +} + +# start redis server or sentinel and check process uid and gid +run_redis_docker_and_check_uid_gid() { + docker_flags= + expected_cmd="redis-server" + user=redis + group=redis + file_owner= + + while [[ $# -gt 0 ]]; do + case "$1" in + --user) + user="$2" + shift 2 + ;; + --group) + group="$2" + shift 2 + ;; + --expected-cmd) + expected_cmd="$2" + shift 2 + ;; + --docker-flags) + docker_flags=$2 + shift 2 + ;; + --file-owner) + file_owner="$2" + shift 2 + ;; + --*) + fail "Unknown flag $1" + return 1 + ;; + *) + break + ;; + esac + done + + if echo "$expected_cmd" | grep -q "sentinel"; then + dir="$(readlink -f "$(mktemp -d -p .)")" + touch "$dir/sentinel.conf" + if [ "$file_owner" ]; then + sudo chown -R "$file_owner" "$dir" + fi + docker_flags="-v $dir:/etc/sentinel $docker_flags" + fi + + docker_cmd="$*" + # shellcheck disable=SC2086 + container=$(docker run $docker_flags -d "$REDIS_IMG" $docker_cmd) + ret=$? + + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + if [ $ret -gt 0 ]; then + echo "retarning" + return 1 + fi + + cmdline=$(docker exec "$container" cat /proc/1/cmdline|tr -d \\0) + assertContains "$docker_flags $docker_cmd, cmdline: $cmdline" "$cmdline" "$expected_cmd" + + redis_user_uid=$(docker exec "$container" id -u "$user") + redis_user_gid=$(docker exec "$container" id -g "$group") + + status=$(docker exec "$container" cat /proc/1/status) + process_uid=$(echo "$status" | grep Uid | cut -f2) + process_gid=$(echo "$status" | grep Gid | cut -f2) + + assertEquals "redis cmd '$docker_cmd', process uid" "$redis_user_uid" "$process_uid" + assertEquals "redis cmd '$docker_cmd', process gid" "$redis_user_gid" "$process_gid" + + docker stop "$container" >/dev/null + if [ "$dir" ]; then + sudo rm -rf "$dir" + fi +} + +run_redis_docker_and_check_modules() { + docker_cmd="$1" + # shellcheck disable=SC2086 + container=$(docker run --rm -d "$REDIS_IMG" $docker_cmd) + info=$(docker exec "$container" redis-cli info) + + [ "$PLATFORM" ] && [ "$PLATFORM" != "amd64" ] && startSkipping + assertContains "$info" "module:name=timeseries" + assertContains "$info" "module:name=search" + assertContains "$info" "module:name=bf" + assertContains "$info" "module:name=vectorset" + assertContains "$info" "module:name=ReJSON" + + docker stop "$container" >/dev/null +} + +# helper assert function to check redis output +assert_redis_output_has_no_config_perm_error() { + s="can't open config file" + assertNotContains "cmd: $docker_cmd, docker output contains '$s': " "$docker_output" "$s" +} + +assert_redis_v8() { + assertContains "$1" "Redis server v=8" +} + +# Tests # + +test_redis_version() { + ret=$(docker run --rm "$REDIS_IMG" -v|tail -n 1) + assert_redis_v8 "$ret" +} + +test_data_dir_owner_and_perms_changed_by_server_when_data_is_RO() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 + appendonlydir dir $HOST_OWNER -> $REDIS_UID 0333 -> 0600 + dump.rdb file $HOST_OWNER -> $REDIS_UID 0333 -> 0600 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_changed_by_server_when_appendonlydir_contains_files() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 + appendonlydir dir $HOST_OWNER -> $REDIS_UID 0333 -> 0600 + appendonlydir/foo.aof dir $HOST_OWNER -> $REDIS_UID 0333 -> 0600 + dump.rdb file $HOST_OWNER -> $REDIS_UID 0333 -> 0600 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_changed_by_server_when_data_is_empty_and_RO() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_not_changed_by_server_when_data_is_RW() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0777 -> 0777 + appendonlydir dir $HOST_OWNER -> $HOST_UID 0666 -> 0666 + dump.rdb file $HOST_OWNER -> $HOST_UID 0666 -> 0666 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_not_changed_by_server_when_data_contains_unknown_file() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + appendonlydir dir $HOST_OWNER -> $HOST_UID 0444 -> 0444 + dump.rdb file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + garbage.file file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_not_changed_by_server_when_data_contains_unknown_subdir() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + somedir dir $HOST_OWNER -> $HOST_UID 0444 -> 0444 + dump.rdb file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_not_changed_when_sentinel() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + appendonlydir dir $HOST_OWNER -> $HOST_UID 0333 -> 0333 + dump.rdb file $HOST_OWNER -> $HOST_UID 0333 -> 0333 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /data) + run_docker_and_test_ownership_with_common_flags_for_sentinel +} + + +test_config_owner_not_changed_by_server_when_config_is_readable() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + redis.conf file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis) + run_docker_and_test_ownership_with_common_flags_for_server /etc/redis/redis.conf +} + +test_only_config_file_owner_and_perms_changed_by_server_when_only_config_is_not_readable() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + redis.conf file $HOST_OWNER -> $REDIS_UID 0000 -> 0400 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis) + run_docker_and_test_ownership_with_common_flags_for_server /etc/redis/redis.conf +} + +test_config_file_and_dir_owner_and_perms_changed_by_server_when_not_readable() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0000 -> 0400 + redis.conf file $HOST_OWNER -> $REDIS_UID 0000 -> 0400 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis) + run_docker_and_test_ownership_with_common_flags_for_server /etc/redis/redis.conf +} + +test_config_owner_and_perms_not_changed_when_unknown_file_exists() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0000 -> 0000 + redis.conf file $HOST_OWNER -> $HOST_UID 0000 -> 0000 + garbage.file file $HOST_OWNER -> $HOST_UID 0000 -> 0000 + " + + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis) + run_docker_and_test_ownership_with_common_flags_for_server /etc/redis/redis.conf + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/redis.conf +} + +test_config_owner_and_perms_not_changed_when_unknown_subdir_exists() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0000 -> 0000 + redis.conf file $HOST_OWNER -> $HOST_UID 0000 -> 0000 + some dir $HOST_OWNER -> $HOST_UID 0000 -> 0000 + " + + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis) + run_docker_and_test_ownership_with_common_flags_for_server /etc/redis/redis.conf + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/redis.conf +} + +test_config_owner_and_perms_not_changed_by_sentinel_when_config_is_RW() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0777 -> 0777 + sentinel.conf file $HOST_OWNER -> $HOST_UID 0666 -> 0666 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis/sentinel) + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/sentinel/sentinel.conf +} + +test_config_file_and_dir_owner_and_perms_changed_by_sentinel_when_RO() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 + sentinel.conf file $HOST_OWNER -> $REDIS_UID 0400 -> 0600 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis/sentinel) + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/sentinel/sentinel.conf +} + +test_config_dir_owner_and_perms_changed_by_sentinel_when_only_dir_is_RO() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0555 -> 0700 + sentinel.conf file $HOST_OWNER -> $HOST_UID 0666 -> 0666 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis/sentinel) + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/sentinel/sentinel.conf +} + +test_config_owner_and_perms_changed_by_sentinel_when_config_is_WO() { + dir_structure=" + . dir $HOST_OWNER -> $REDIS_UID 0333 -> 0700 + sentinel.conf file $HOST_OWNER -> $REDIS_UID 0222 -> 0600 + " + common_flags=(--dir-structure "$dir_structure" --mount-target /etc/redis/sentinel) + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/sentinel/sentinel.conf +} + +# test that entrypoint tries to start redis even when config is non existent dir +test_redis_start_reached_when_config_dir_does_not_exist() { + assert_has_config_error() { + # shellcheck disable=SC2317 + assertContains "$docker_output" "Fatal error, can't open config file" + # shellcheck disable=SC2317 + assertContains "$docker_output" "No such file or directory" + } + common_flags=(--mount-target /etc/somewhere --extra-assert assert_has_config_error) + run_docker_and_test_ownership_with_common_flags_for_server /etc/nowhere/redis.conf +} + +test_redis_start_reached_when_chown_on_data_dir_is_denied() { + assert_internal() { + # shellcheck disable=SC2317 + assert_redis_v8 "$docker_output" + } + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0333 -> 0700 + dump.rdb file root:root -> 0 0222 -> 0222 + " + common_flags=(--mount-target /data + --dir-structure "$dir_structure" + --extra-assert assert_internal + --docker-flags "--cap-drop=chown" + ) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_data_dir_owner_and_perms_not_changed_by_server_when_data_is_RO_and_SKIP_FIX_PERMS_is_used() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + datum.rdb file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + " + common_flags=(--mount-target /data + --dir-structure "$dir_structure" + --docker-flags "-e SKIP_FIX_PERMS=1" + ) + run_docker_and_test_ownership_with_common_flags_for_server +} + +test_config_owner_and_perms_not_changed_by_sentinel_when_config_is_RO_and_SKIP_FIX_PERMS_is_used() { + dir_structure=" + . dir $HOST_OWNER -> $HOST_UID 0555 -> 0555 + sentinel.conf file $HOST_OWNER -> $HOST_UID 0444 -> 0444 + " + common_flags=(--mount-target /etc/redis/sentinel + --dir-structure "$dir_structure" + --docker-flags "-e SKIP_FIX_PERMS=1" + ) + run_docker_and_test_ownership_with_common_flags_for_sentinel /etc/redis/sentinel/sentinel.conf +} + + + +test_redis_server_persistence_with_bind_mount() { + dir=$(mktemp -d -p .) + + # make data directory non writable + chmod 0444 "$dir" + + container=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG" --appendonly yes) + + result=$(echo save | docker exec -i "$container" redis-cli) + assertEquals "OK" "$result" + + # save container hash as a value + result=$(echo "SET FOO $container" | docker exec -i "$container" redis-cli) + assertEquals "OK" "$result" + + docker stop "$container" >/dev/null + + # change the owner + sudo chown -R "$HOST_OWNER" "$dir" + + container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) + assertEquals "$container" "$value" + + docker stop "$container2" >/dev/null + + sudo rm -rf "$dir" +} + +test_redis_server_persistence_with_volume() { + docker volume rm test_redis >/dev/null 2>&1 || : + + docker volume create test_redis >/dev/null + + # change owner of the data volume + docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data' + + container=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG" --appendonly yes) + + result=$(echo save | docker exec -i "$container" redis-cli) + assertEquals "OK" "$result" + + # save container hash as a value + result=$(echo "SET FOO $container" | docker exec -i "$container" redis-cli) + assertEquals "OK" "$result" + + docker stop "$container" >/dev/null + + # change owner and permissions of files in data volume + docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data && chmod 0000 -R /data' + + container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) + assertEquals "$container" "$value" + + docker stop "$container2" >/dev/null + + docker volume rm test_redis >/dev/null || : +} + +test_redis_process_uid_and_gid_are_redis() { + run_redis_docker_and_check_uid_gid "" + run_redis_docker_and_check_uid_gid redis-server + run_redis_docker_and_check_uid_gid /usr/local/bin/redis-server + + run_redis_docker_and_check_uid_gid --expected-cmd redis-sentinel redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid --expected-cmd redis-sentinel /usr/local/bin/redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid --expected-cmd "[sentinel]" /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid --expected-cmd "[sentinel]" redis-server /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid --expected-cmd "[sentinel]" /usr/local/bin/redis-server /etc/sentinel/sentinel.conf --sentinel +} + +test_redis_process_uid_and_gid_respects_docker_user_arg() { + read -r daemon_user_uid _ <<< "$(get_container_user_uid_gid_on_the_host daemon:daemon)" + + # disable persistence as directory data dir would not be writable + common_flags=(--user daemon --group daemon --docker-flags "--user daemon") + run_redis_docker_and_check_uid_gid "${common_flags[@]}" "" --save "" + run_redis_docker_and_check_uid_gid "${common_flags[@]}" redis-server --save "" + run_redis_docker_and_check_uid_gid "${common_flags[@]}" /usr/local/bin/redis-server --save "" + + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --file-owner "$daemon_user_uid" --expected-cmd redis-sentinel redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --file-owner "$daemon_user_uid" --expected-cmd redis-sentinel /usr/local/bin/redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --file-owner "$daemon_user_uid" --expected-cmd "[sentinel]" /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --file-owner "$daemon_user_uid" --expected-cmd "[sentinel]" redis-server /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --file-owner "$daemon_user_uid" --expected-cmd "[sentinel]" /usr/local/bin/redis-server /etc/sentinel/sentinel.conf --sentinel +} + +test_redis_process_uid_and_gid_are_root_when_SKIP_DROP_PRIVS_is_used() { + common_flags=(--user root --group root --docker-flags "-e SKIP_DROP_PRIVS=1") + run_redis_docker_and_check_uid_gid "${common_flags[@]}" "" --save "" + run_redis_docker_and_check_uid_gid "${common_flags[@]}" redis-server --save "" + run_redis_docker_and_check_uid_gid "${common_flags[@]}" /usr/local/bin/redis-server --save "" + + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --expected-cmd redis-sentinel redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --expected-cmd redis-sentinel /usr/local/bin/redis-sentinel /etc/sentinel/sentinel.conf + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --expected-cmd "[sentinel]" /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --expected-cmd "[sentinel]" redis-server /etc/sentinel/sentinel.conf --sentinel + run_redis_docker_and_check_uid_gid "${common_flags[@]}" --expected-cmd "[sentinel]" /usr/local/bin/redis-server /etc/sentinel/sentinel.conf --sentinel +} + +test_redis_server_modules_are_loaded() { + run_redis_docker_and_check_modules + run_redis_docker_and_check_modules redis-server + run_redis_docker_and_check_modules /usr/local/bin/redis-server +} + +# shellcheck disable=SC1091 +. ./shunit2 diff --git a/test/shunit2 b/test/shunit2 new file mode 100755 index 000000000..7b7c7c199 --- /dev/null +++ b/test/shunit2 @@ -0,0 +1,1612 @@ +#! /bin/sh +# vim:et:ft=sh:sts=2:sw=2 +# +# shUnit2 -- Unit testing framework for Unix shell scripts. +# +# Copyright 2008-2021 Kate Ward. All Rights Reserved. +# Released under the Apache 2.0 license. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Author: kate.ward@forestent.com (Kate Ward) +# https://github.com/kward/shunit2 +# +# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is +# based on the popular JUnit unit testing framework for Java. +# +# `expr` may be antiquated, but it is the only solution in some cases. +# shellcheck disable=SC2003 +# Allow usage of legacy backticked `...` notation instead of $(...). +# shellcheck disable=SC2006 + +# Return if shunit2 already loaded. +if test -n "${SHUNIT_VERSION:-}"; then + exit 0 +fi +SHUNIT_VERSION='2.1.9pre' + +# Return values that scripts can use. +SHUNIT_TRUE=0 +SHUNIT_FALSE=1 +SHUNIT_ERROR=2 + +# Determine if `builtin` command exists. +__SHUNIT_BUILTIN='builtin' +# shellcheck disable=2039 +if ! ("${__SHUNIT_BUILTIN}" echo 123 >/dev/null 2>&1); then + __SHUNIT_BUILTIN='' +fi + +# Determine some reasonable command defaults. +__SHUNIT_CMD_ECHO_ESC='echo -e' +# shellcheck disable=SC2039,SC3037 +if ${__SHUNIT_BUILTIN} [ "`echo -e test`" = '-e test' ]; then + __SHUNIT_CMD_ECHO_ESC='echo' +fi + +# Determine if `date` supports nanosecond resolution. +__SHUNIT_CMD_DATE_SECONDS='date +%s.%N' +if ${__SHUNIT_BUILTIN} [ "`date +%N`" = '%N' ]; then + __SHUNIT_CMD_DATE_SECONDS='date +%s' +fi + +# Determine `bc` command. +__SHUNIT_CMD_BC='bc' +if ! (${__SHUNIT_CMD_BC} --help >/dev/null 2>&1); then + __SHUNIT_CMD_BC='busybox bc' +fi +if ! (${__SHUNIT_CMD_BC} --help >/dev/null 2>&1); then + __SHUNIT_CMD_BC='' +fi + +# Determine `dc` command. +__SHUNIT_CMD_DC='dc' +if ! (${__SHUNIT_CMD_DC} --help >/dev/null 2>&1); then + __SHUNIT_CMD_DC='busybox dc' +fi +if ! (${__SHUNIT_CMD_DC} --help >/dev/null 2>&1); then + __SHUNIT_CMD_DC='' +fi + +# Format float numbers to the single style from different tools. +# Args: +# num: string: float number to format +# Returns: +# string: formatted number. Empty string if error occurs. +_shunit_float_format() { + # Double-dot number is an error. + # No need to format if the number is an integer. + case "${1}" in + *.*.*) + return + ;; + *.*) + ;; + *) + echo "${1}" + return + ;; + esac + + _shunit_format_result_="$1" + + # Add leading zero if needed. + _shunit_format_result_="$(echo "${_shunit_format_result_}" \ + |command sed 's/^\./0./g')" + + # Remove trailing zeros. + _shunit_format_result_="$(echo "${_shunit_format_result_}" \ + |command sed 's/0\+$//g')" + + # Remove trailing dot. + _shunit_format_result_="$(echo "${_shunit_format_result_}" \ + |command sed 's/\.$//g')" + + # Print the result. + echo "${_shunit_format_result_}" + unset _shunit_format_result_ +} + +# Calculate numbers using bc. +# Args: +# left: string: left operand (may be float point) +# operation: string: operation (+ - * /) +# right: string: right operand (may be float point) +# Returns: +# string: result +_shunit_calc_bc() { + _shunit_output_="$(echo "$@" \ + |command ${__SHUNIT_CMD_BC:?})" + shunit_return=$? + if ${__SHUNIT_BUILTIN} [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_float_format "${_shunit_output_}" + shunit_return=$? + fi + + unset _shunit_output_ + return ${shunit_return} +} + +# Calculate numbers using dc. +# Args: +# left: string: left operand (may be float point) +# operation: string: operation (+ - * /) +# right: string: right operand (may be float point) +# Returns: +# string: result +_shunit_calc_dc() { + _shunit_output_="$(echo "$1" "$3" "$2" "p" \ + |command ${__SHUNIT_CMD_DC:?})" + shunit_return=$? + if ${__SHUNIT_BUILTIN} [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_float_format "${_shunit_output_}" + shunit_return=$? + fi + + unset _shunit_output_ + return ${shunit_return} +} + +# Calculate numbers using expr. +# Args: +# left: string: left integer number operand +# operation: string: operation (+ - * /) +# right: string: right integer number operand +# Returns: +# string: result. Empty string if error occurs. +_shunit_calc_expr() { + expr "$@" 2>/dev/null || ${__SHUNIT_BUILTIN} true +} + +# Determine what command to use for calculating numbers. +__SHUNIT_CMD_CALC='_shunit_calc_bc' +if ! ("${__SHUNIT_CMD_CALC}" 1 + 2 >/dev/null 2>&1); then + __SHUNIT_CMD_CALC=_shunit_calc_dc +fi +if ! ("${__SHUNIT_CMD_CALC}" 1 + 2 >/dev/null 2>&1); then + __SHUNIT_CMD_CALC=_shunit_calc_expr +fi + +# Commands a user can override if needed. +__SHUNIT_CMD_TPUT='tput' +SHUNIT_CMD_TPUT=${SHUNIT_CMD_TPUT:-${__SHUNIT_CMD_TPUT}} + +# Enable color output. Options are 'auto', 'always', or 'never'. +SHUNIT_COLOR=${SHUNIT_COLOR:-auto} + +# +# Internal constants. +# + +__SHUNIT_MODE_SOURCED='sourced' +__SHUNIT_MODE_STANDALONE='standalone' +__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} + +# User provided test prefix to display in front of the name of the test being +# executed. Define by setting the SHUNIT_TEST_PREFIX variable. +__SHUNIT_TEST_PREFIX=${SHUNIT_TEST_PREFIX:-} + +# ANSI colors. +__SHUNIT_ANSI_NONE='\033[0m' +__SHUNIT_ANSI_RED='\033[1;31m' +__SHUNIT_ANSI_GREEN='\033[1;32m' +__SHUNIT_ANSI_YELLOW='\033[1;33m' +__SHUNIT_ANSI_CYAN='\033[1;36m' + +# +# Internal variables. +# + +# Variables. +__shunit_lineno='' # Line number of executed test. +__shunit_mode=${__SHUNIT_MODE_SOURCED} # Operating mode. +__shunit_reportGenerated=${SHUNIT_FALSE} # Is report generated. +__shunit_script='' # Filename of unittest script (standalone mode). +__shunit_skip=${SHUNIT_FALSE} # Is skipping enabled. +__shunit_suite='' # Suite of tests to execute. +__shunit_clean=${SHUNIT_FALSE} # _shunit_cleanup() was already called. +__shunit_suiteName='' # Text name of current test suite. +__shunit_xmlSuiteName='' # XML-ready text name of current test suite. + +# JUnit XML variables. +__shunit_junitXmlOutputFile='' # File to use for JUnit XML output in addition to stdout. +__shunit_junitXmlTestCases='' # Test cases info in the JUnit XML format for output +__shunit_junitXmlCurrentTestCaseErrors='' # Current test case error info in the JUnit XML format for output + +# Time variables +__shunit_startSuiteTime='' # When the suite execution was started +__shunit_endSuiteTime='' # When the suite execution ended +__shunit_startCaseTime='' # When the case execution was started +__shunit_endCaseTime='' # When the case execution ended + +# ANSI colors (populated by _shunit_configureColor()). +__shunit_ansi_none='' +__shunit_ansi_red='' +__shunit_ansi_green='' +__shunit_ansi_yellow='' +__shunit_ansi_cyan='' + +# Counts of tests. +__shunit_testSuccess=${SHUNIT_TRUE} +__shunit_testsTotal=0 +__shunit_testsPassed=0 +__shunit_testsFailed=0 + +# Counts of asserts. +__shunit_assertsTotal=0 +__shunit_assertsPassed=0 +__shunit_assertsFailed=0 +__shunit_assertsSkipped=0 +__shunit_assertsCurrentTest=0 + +# +# Internal functions. +# + +# Logging. +_shunit_warn() { + ${__SHUNIT_CMD_ECHO_ESC} "${__shunit_ansi_yellow}shunit2:WARN${__shunit_ansi_none} $*" >&2 +} +_shunit_error() { + ${__SHUNIT_CMD_ECHO_ESC} "${__shunit_ansi_red}shunit2:ERROR${__shunit_ansi_none} $*" >&2 +} +_shunit_fatal() { + ${__SHUNIT_CMD_ECHO_ESC} "${__shunit_ansi_red}shunit2:FATAL${__shunit_ansi_none} $*" >&2 + exit ${SHUNIT_ERROR} +} + +# +# Macros. +# + +# shellcheck disable=SC2016,SC2089 +_SHUNIT_LINENO_='eval __shunit_lineno=""; if ${__SHUNIT_BUILTIN} [ "${1:-}" = "--lineno" ] && ${__SHUNIT_BUILTIN} [ -n "${2:-}" ]; then __shunit_lineno="[${2}]"; shift 2; fi;' + +# +# Setup. +# + +# Specific shell checks. +if ${__SHUNIT_BUILTIN} [ -n "${ZSH_VERSION:-}" ]; then + setopt |grep "^shwordsplit$" >/dev/null + if ${__SHUNIT_BUILTIN} [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_fatal 'zsh shwordsplit option is required for proper operation' + fi + if ${__SHUNIT_BUILTIN} [ -z "${SHUNIT_PARENT:-}" ]; then + _shunit_fatal "zsh does not pass \$0 through properly. please declare \ +\"SHUNIT_PARENT=\$0\" before calling shUnit2" + fi +fi + +# Set the constants readonly. +__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` +echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ + __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` +for __shunit_const in ${__shunit_constants}; do + if ${__SHUNIT_BUILTIN} [ -z "${ZSH_VERSION:-}" ]; then + readonly "${__shunit_const}" + else + case ${ZSH_VERSION} in + [123].*) readonly "${__shunit_const}" ;; + *) + # Declare readonly constants globally. + # shellcheck disable=SC2039,SC3045 + readonly -g "${__shunit_const}" + esac + fi +done +unset __shunit_const __shunit_constants + +#----------------------------------------------------------------------------- +# Assertion functions. +# + +# Assert that two values are equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertEquals() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if ${__SHUNIT_BUILTIN} [ "${shunit_expected_}" = "${shunit_actual_}" ]; then + _shunit_assertPass + else + failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' + +# Assert that two values are not equal to one another. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotEquals() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if ${__SHUNIT_BUILTIN} [ "${shunit_expected_}" != "${shunit_actual_}" ]; then + _shunit_assertPass + else + failSame "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' + +# Assert that a container contains a content. +# +# Args: +# message: string: failure message [optional] +# container: string: container to analyze +# content: string: content to find +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertContains() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertContains() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_container_=$1 + shunit_content_=$2 + shunit_return=${SHUNIT_TRUE} + if echo "${shunit_container_}" |grep -F -- "${shunit_content_}" >/dev/null; then + _shunit_assertPass + else + failNotFound "${shunit_message_}" "${shunit_content_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_container_ shunit_content_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_CONTAINS_='eval assertContains --lineno "${LINENO:-}"' + +# Assert that a container does not contain a content. +# +# Args: +# message: string: failure message [optional] +# container: string: container to analyze +# content: string: content to look for +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotContains() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotContains() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_container_=$1 + shunit_content_=$2 + + shunit_return=${SHUNIT_TRUE} + if echo "$shunit_container_" |grep -F -- "$shunit_content_" > /dev/null; then + failFound "${shunit_message_}" "${shunit_content_}" + shunit_return=${SHUNIT_FALSE} + else + _shunit_assertPass + fi + + unset shunit_message_ shunit_container_ shunit_content_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_CONTAINS_='eval assertNotContains --lineno "${LINENO:-}"' + +# Assert that a value is null (i.e. an empty string). +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNull() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -gt 2 ]; then + # Allowing 0 arguments as $1 might actually be null. + _shunit_error "assertNull() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + ${__SHUNIT_BUILTIN} test -z "${1:-}" + assertTrue "${shunit_message_}" $? + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' + +# Assert that a value is not null (i.e. a non-empty string). +# +# Args: +# message: string: failure message [optional] +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotNull() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -gt 2 ]; then + # Allowing 0 arguments as $1 might actually be null. + _shunit_error "assertNotNull() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + ${__SHUNIT_BUILTIN} test -n "${1:-}" + assertTrue "${shunit_message_}" $? + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' + +# Assert that two values are the same (i.e. equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertSame() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' + +# Assert that two values are not the same (i.e. not equal to one another). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertNotSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotSame() requires two or three arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_:-}$1" + shift + fi + assertNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is true. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertTrue 0 +# assertTrue "[ 34 -gt 23 ]" +# The following test will fail with a message: +# assertTrue 123 +# assertTrue "test failed" "[ -r '/non/existent/file' ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertTrue() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertTrue() takes one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # See if condition is an integer, i.e. a return value. + shunit_return=${SHUNIT_TRUE} + if ${__SHUNIT_BUILTIN} [ -z "${shunit_condition_}" ]; then + # Null condition. + shunit_return=${SHUNIT_FALSE} + elif (expr \( "${shunit_condition_}" + '0' \) '=' "${shunit_condition_}" >/dev/null 2>&1) + then + # Possible return value. Treating 0 as true, and non-zero as false. + if ${__SHUNIT_BUILTIN} [ "${shunit_condition_}" -ne 0 ]; then + shunit_return=${SHUNIT_FALSE} + fi + else + # Hopefully... a condition. + if ! eval "${shunit_condition_}" >/dev/null 2>&1; then + shunit_return=${SHUNIT_FALSE} + fi + fi + + # Record the test. + if ${__SHUNIT_BUILTIN} [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' + +# Assert that a value or shell test condition is false. +# +# In shell, a value of 0 is true and a non-zero value is false. Any integer +# value passed can thereby be tested. +# +# Shell supports much more complicated tests though, and a means to support +# them was needed. As such, this function tests that conditions are true or +# false through evaluation rather than just looking for a true or false. +# +# The following test will succeed: +# assertFalse 1 +# assertFalse "[ 'apples' = 'oranges' ]" +# The following test will fail with a message: +# assertFalse 0 +# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" +# +# Args: +# message: string: failure message [optional] +# condition: string: integer value or shell conditional statement +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +assertFalse() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertFalse() requires one or two arguments; $# given" + _shunit_assertFail + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # See if condition is an integer, i.e. a return value. + shunit_return=${SHUNIT_TRUE} + if ${__SHUNIT_BUILTIN} [ -z "${shunit_condition_}" ]; then + # Null condition. + shunit_return=${SHUNIT_TRUE} + elif (expr \( "${shunit_condition_}" + '0' \) '=' "${shunit_condition_}" >/dev/null 2>&1); then + # Possible return value. Treating 0 as true, and non-zero as false. + if ${__SHUNIT_BUILTIN} [ "${shunit_condition_}" -eq 0 ]; then + shunit_return=${SHUNIT_FALSE} + fi + else + # Hopefully... a condition. + # shellcheck disable=SC2086 + if eval ${shunit_condition_} >/dev/null 2>&1; then + shunit_return=${SHUNIT_FALSE} + fi + fi + + # Record the test. + if ${__SHUNIT_BUILTIN} [ "${shunit_return}" -eq "${SHUNIT_TRUE}" ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ + return "${shunit_return}" +} +# shellcheck disable=SC2016,SC2034 +_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# Failure functions. +# + +# Records a test failure. +# +# Args: +# message: string: failure message [optional] +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +fail() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -gt 1 ]; then + _shunit_error "fail() requires zero or one arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 1 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_}" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_='eval fail --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotEquals() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' + +# Records a test failure, stating a value was found. +# +# Args: +# message: string: failure message [optional] +# content: string: found value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failFound() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "failFound() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_content_=$1 + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }found:<${shunit_content_}>" + + unset shunit_message_ shunit_content_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_FOUND_='eval failFound --lineno "${LINENO:-}"' + +# Records a test failure, stating a content was not found. +# +# Args: +# message: string: failure message [optional] +# content: string: content not found +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotFound() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "failNotFound() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_content_=$1 + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }not found:<${shunit_content_}>" + + unset shunit_message_ shunit_content_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_FOUND_='eval failNotFound --lineno "${LINENO:-}"' + +# Records a test failure, stating two values should have been the same. +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + shunit_message_=${shunit_message_%% } + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' + +# Records a test failure, stating two values were not equal. +# +# This is functionally equivalent to calling failNotEquals(). +# +# Args: +# message: string: failure message [optional] +# expected: string: expected value +# actual: string: actual value +# Returns: +# integer: success (TRUE/FALSE/ERROR constant) +failNotSame() { + # shellcheck disable=SC2090 + ${_SHUNIT_LINENO_} + if ${__SHUNIT_BUILTIN} [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotSame() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + if _shunit_shouldSkip; then + return ${SHUNIT_TRUE} + fi + + shunit_message_=${__shunit_lineno} + if ${__SHUNIT_BUILTIN} [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + failNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +# shellcheck disable=SC2016,SC2034 +_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# Skipping functions. +# + +# Force remaining assert and fail functions to be "skipped". +# +# This function forces the remaining assert and fail functions to be "skipped", +# i.e. they will have no effect. Each function skipped will be recorded so that +# the total of asserts and fails will not be altered. +# +# Args: +# message: string: message to provide to user [optional] +startSkipping() { + if ${__SHUNIT_BUILTIN} [ $# -gt 0 ]; then _shunit_warn "[skipping] $*"; fi + __shunit_skip=${SHUNIT_TRUE} +} + +# Resume the normal recording behavior of assert and fail calls. +# +# Args: +# None +endSkipping() { __shunit_skip=${SHUNIT_FALSE}; } + +# Returns the state of assert and fail call skipping. +# +# Args: +# None +# Returns: +# boolean: (TRUE/FALSE constant) +isSkipping() { return ${__shunit_skip}; } + +#----------------------------------------------------------------------------- +# Suite functions. +# + +# Stub. This function should contains all unit test calls to be made. +# +# DEPRECATED (as of 2.1.0) +# +# This function can be optionally overridden by the user in their test suite. +# +# If this function exists, it will be called when shunit2 is sourced. If it +# does not exist, shunit2 will search the parent script for all functions +# beginning with the word 'test', and they will be added dynamically to the +# test suite. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Adds a function name to the list of tests schedule for execution. +# +# This function should only be called from within the suite() function. +# +# Args: +# function: string: name of a function to add to current unit test suite +suite_addTest() { + shunit_func_=${1:-} + + __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" + __shunit_testsTotal=`expr "${__shunit_testsTotal}" + 1` + + unset shunit_func_ +} + +# Stub. This function will be called once before any tests are run. +# +# Common one-time environment preparation tasks shared by all tests can be +# defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called once after all tests are finished. +# +# Common one-time environment cleanup tasks shared by all tests can be defined +# here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Stub. This function will be called before each test is run. +# +# Common environment preparation tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#setUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +# Note: see _shunit_mktempFunc() for actual implementation +# Stub. This function will be called after each test is run. +# +# Common environment cleanup tasks shared by all tests can be defined here. +# +# This function should be overridden by the user in their unit test suite. +# Note: see _shunit_mktempFunc() for actual implementation +# +# Args: +# None +#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION + +#------------------------------------------------------------------------------ +# Internal shUnit2 functions. +# + +# Create a temporary directory to store various run-time files in. +# +# This function is a cross-platform temporary directory creation tool. Not all +# OSes have the `mktemp` function, so one is included here. +# +# Args: +# None +# Outputs: +# string: the temporary directory that was created +_shunit_mktempDir() { + # Try the standard `mktemp` function. + if ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ); then + return + fi + + # The standard `mktemp` didn't work. Use our own. + # shellcheck disable=SC2039,SC3028 + if ${__SHUNIT_BUILTIN} [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then + _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" +#! /bin/sh +exit ${SHUNIT_TRUE} +EOF + command chmod +x "${_shunit_file_}" + done + + unset _shunit_file_ +} + +# Final cleanup function to leave things as we found them. +# +# Besides removing the temporary directory, this function is in charge of the +# final exit code of the unit test. The exit code is based on how the script +# was ended (e.g. normal exit, or via Ctrl-C). +# +# Args: +# name: string: name of the trap called (specified when trap defined) +_shunit_cleanup() { + _shunit_name_=$1 + + _shunit_signal_=0 + case "${_shunit_name_}" in + EXIT) ;; + INT) _shunit_signal_=130 ;; # 2+128 + TERM) _shunit_signal_=143 ;; # 15+128 + *) + _shunit_error "unrecognized trap value (${_shunit_name_})" + ;; + esac + if ${__SHUNIT_BUILTIN} [ "${_shunit_name_}" != 'EXIT' ]; then + _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" + fi + + # Do our work. + if ${__SHUNIT_BUILTIN} [ ${__shunit_clean} -eq ${SHUNIT_FALSE} ]; then + # Ensure tear downs are only called once. + __shunit_clean=${SHUNIT_TRUE} + + tearDown || _shunit_warn 'tearDown() returned non-zero return code.' + __shunit_endCaseTime=`${__SHUNIT_CMD_DATE_SECONDS}` + oneTimeTearDown || \ + _shunit_warn 'oneTimeTearDown() returned non-zero return code.' + __shunit_endSuiteTime=`${__SHUNIT_CMD_DATE_SECONDS}` + + command rm -fr "${__shunit_tmpDir}" + fi + + if ${__SHUNIT_BUILTIN} [ "${_shunit_name_}" != 'EXIT' ]; then + # Handle all non-EXIT signals. + trap - 0 # Disable EXIT trap. + exit ${_shunit_signal_} + elif ${__SHUNIT_BUILTIN} [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ]; then + _shunit_assertFail 'unknown failure encountered running a test' + _shunit_generateReport + exit ${SHUNIT_ERROR} + fi + + unset _shunit_name_ _shunit_signal_ +} + +# configureColor based on user color preference. +# +# Args: +# color: string: color mode (one of `always`, `auto`, or `never`). +_shunit_configureColor() { + _shunit_color_=${SHUNIT_FALSE} # By default, no color. + case $1 in + 'always') _shunit_color_=${SHUNIT_TRUE} ;; + 'auto') + if ${__SHUNIT_BUILTIN} [ "`_shunit_colors`" -ge 8 ]; then + _shunit_color_=${SHUNIT_TRUE} + fi + ;; + 'never'|'none') ;; # Support 'none' to support legacy usage. + *) _shunit_fatal "unrecognized color option '$1'" ;; + esac + + # shellcheck disable=SC2254 + case ${_shunit_color_} in + ${SHUNIT_TRUE}) + __shunit_ansi_none=${__SHUNIT_ANSI_NONE} + __shunit_ansi_red=${__SHUNIT_ANSI_RED} + __shunit_ansi_green=${__SHUNIT_ANSI_GREEN} + __shunit_ansi_yellow=${__SHUNIT_ANSI_YELLOW} + __shunit_ansi_cyan=${__SHUNIT_ANSI_CYAN} + ;; + ${SHUNIT_FALSE}) + __shunit_ansi_none='' + __shunit_ansi_red='' + __shunit_ansi_green='' + __shunit_ansi_yellow='' + __shunit_ansi_cyan='' + ;; + esac + + unset _shunit_color_ _shunit_tput_ +} + +# colors returns the number of supported colors for the TERM. +_shunit_colors() { + if _shunit_tput_=`${SHUNIT_CMD_TPUT} colors 2>/dev/null`; then + echo "${_shunit_tput_}" + else + echo 16 + fi + unset _shunit_tput_ +} + +# The actual running of the tests happens here. +# +# Args: +# None +_shunit_execSuite() { + for _shunit_test_ in ${__shunit_suite}; do + __shunit_testSuccess=${SHUNIT_TRUE} + + # Reset per-test info + __shunit_assertsCurrentTest=0 + __shunit_junitXmlCurrentTestCaseErrors='' + + # Disable skipping. + endSkipping + + __shunit_startCaseTime=`${__SHUNIT_CMD_DATE_SECONDS}` + + # Execute the per-test setUp() function. + if ! setUp; then + _shunit_fatal "setUp() returned non-zero return code." + fi + + # Execute the test. + echo "${__SHUNIT_TEST_PREFIX}${_shunit_test_}" + # shellcheck disable=SC2086 + if ! eval ${_shunit_test_}; then + _shunit_error "${_shunit_test_}() returned non-zero return code." + __shunit_testSuccess=${SHUNIT_ERROR} + fi + + # Execute the per-test tearDown() function. + if ! tearDown; then + _shunit_fatal "tearDown() returned non-zero return code." + fi + __shunit_endCaseTime=`${__SHUNIT_CMD_DATE_SECONDS}` + + _shunit_test_execution_time_=`"${__SHUNIT_CMD_CALC}" "${__shunit_endCaseTime}" - "${__shunit_startCaseTime}"` + + # Store current test case info in JUnit XML. + __shunit_junitXmlTestCases="${__shunit_junitXmlTestCases} + ${__shunit_junitXmlCurrentTestCaseErrors} + " + + # Update stats. + if ${__SHUNIT_BUILTIN} [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then + __shunit_testsPassed=`expr "${__shunit_testsPassed}" + 1` + else + __shunit_testsFailed=`expr "${__shunit_testsFailed}" + 1` + fi + done + + unset _shunit_test_ _shunit_test_execution_time_ +} + +# Generates the user friendly report with appropriate OK/FAILED message. +# +# Args: +# None +# Output: +# string: the report of successful and failed tests, as well as totals. +_shunit_generateReport() { + if ${__SHUNIT_BUILTIN} [ "${__shunit_reportGenerated}" -eq ${SHUNIT_TRUE} ]; then + return + fi + + _shunit_ok_=${SHUNIT_TRUE} + + # If no exit code was provided, determine an appropriate one. + if ${__SHUNIT_BUILTIN} [ "${__shunit_testsFailed}" -gt 0 -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ]; then + _shunit_ok_=${SHUNIT_FALSE} + fi + + echo + _shunit_msg_="Ran ${__shunit_ansi_cyan}${__shunit_testsTotal}${__shunit_ansi_none}" + if ${__SHUNIT_BUILTIN} [ "${__shunit_testsTotal}" -eq 1 ]; then + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} test." + else + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} tests." + fi + + if ${__SHUNIT_BUILTIN} [ -n "${__shunit_junitXmlOutputFile}" ]; then + # Calculate total execution time in seconds. + _shunit_suite_execution_time_=`"${__SHUNIT_CMD_CALC}" "${__shunit_endSuiteTime}" - "${__shunit_startSuiteTime}"` + + # Generate a ISO-8601 compliant date. + _shunit_suite_start_time_preformatted_=`date -u '+%Y-%m-%dT%H:%M:%S%z' -d "@${__shunit_startSuiteTime}"` + + echo " +${__shunit_junitXmlTestCases} +" > "${__shunit_junitXmlOutputFile}" + echo + echo "JUnit XML file ${__shunit_junitXmlOutputFile} was saved." + fi + + if ${__SHUNIT_BUILTIN} [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then + _shunit_msg_="${__shunit_ansi_green}OK${__shunit_ansi_none}" + if ${__SHUNIT_BUILTIN} [ "${__shunit_assertsSkipped}" -gt 0 ]; then + _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none})" + fi + else + _shunit_msg_="${__shunit_ansi_red}FAILED${__shunit_ansi_none}" + _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_red}failures=${__shunit_assertsFailed}${__shunit_ansi_none}" + if ${__SHUNIT_BUILTIN} [ "${__shunit_assertsSkipped}" -gt 0 ]; then + _shunit_msg_="${_shunit_msg_},${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none}" + fi + _shunit_msg_="${_shunit_msg_})" + fi + + echo + ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_}" + __shunit_reportGenerated=${SHUNIT_TRUE} + + unset _shunit_msg_ _shunit_ok_ _shunit_suite_execution_time_ _shunit_suite_start_time_preformatted_ +} + +# Test for whether a function should be skipped. +# +# Args: +# None +# Returns: +# boolean: whether the test should be skipped (TRUE/FALSE constant) +_shunit_shouldSkip() { + if ${__SHUNIT_BUILTIN} test ${__shunit_skip} -eq ${SHUNIT_FALSE}; then + return ${SHUNIT_FALSE} + fi + _shunit_assertSkip +} + +# Records a successful test. +# +# Args: +# None +_shunit_assertPass() { + __shunit_assertsPassed=`expr "${__shunit_assertsPassed}" + 1` + __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` + __shunit_assertsCurrentTest=`expr "${__shunit_assertsCurrentTest}" + 1` +} + +# Records a test failure. +# +# Args: +# message: string: failure message to provide user +_shunit_assertFail() { + __shunit_testSuccess=${SHUNIT_FALSE} + _shunit_incFailedCount + + _shunit_xml_message_="`_shunit_escapeXmlData "$@"`" + + __shunit_junitXmlCurrentTestCaseErrors="${__shunit_junitXmlCurrentTestCaseErrors} + " + + if ${__SHUNIT_BUILTIN} [ $# -gt 0 ]; then + ${__SHUNIT_CMD_ECHO_ESC} "${__shunit_ansi_red}ASSERT:${__shunit_ansi_none}$*" + fi + + unset _shunit_xml_message_ +} + +# Increment the count of failed asserts. +# +# Args: +# none +_shunit_incFailedCount() { + __shunit_assertsFailed=`expr "${__shunit_assertsFailed}" + 1` + __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` + __shunit_assertsCurrentTest=`expr "${__shunit_assertsCurrentTest}" + 1` +} + +# Records a skipped test. +# +# Args: +# None +_shunit_assertSkip() { + __shunit_assertsSkipped=`expr "${__shunit_assertsSkipped}" + 1` + __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` + __shunit_assertsCurrentTest=`expr "${__shunit_assertsCurrentTest}" + 1` +} + +# Dump the current test metrics. +# +# Args: +# none +_shunit_metrics() { + echo "< \ +total: ${__shunit_assertsTotal} \ +passed: ${__shunit_assertsPassed} \ +failed: ${__shunit_assertsFailed} \ +skipped: ${__shunit_assertsSkipped} \ +>" +} + +# Prepare a script filename for sourcing. +# +# Args: +# script: string: path to a script to source +# Returns: +# string: filename prefixed with ./ (if necessary) +_shunit_prepForSourcing() { + _shunit_script_=$1 + case "${_shunit_script_}" in + /*|./*) echo "${_shunit_script_}" ;; + *) echo "./${_shunit_script_}" ;; + esac + unset _shunit_script_ +} + +# Extract list of functions to run tests against. +# +# Args: +# script: string: name of script to extract functions from +# Returns: +# string: of function names +_shunit_extractTestFunctions() { + _shunit_script_=$1 + + # Extract the lines with test function names, strip of anything besides the + # function name, and output everything on a single line. + _shunit_regex_='^\s*((function test[A-Za-z0-9_-]*)|(test[A-Za-z0-9_-]* *\(\)))' + grep -E "${_shunit_regex_}" "${_shunit_script_}" \ + |command sed 's/^[^A-Za-z0-9_-]*//;s/^function //;s/\([A-Za-z0-9_-]*\).*/\1/g' \ + |xargs + + unset _shunit_regex_ _shunit_script_ +} + +# Escape XML data. +# +# Args: +# data: string: data to escape +# Returns: +# string: escaped data +_shunit_escapeXmlData() { + # Required XML characters to escape are described here: + # http://www.w3.org/TR/REC-xml/#syntax + # https://www.liquid-technologies.com/Reference/Glossary/XML_EscapingData.html + echo "$*" \ + |command sed 's/&/\&/g;s//\>/g;s/"/\"/g'";s/'/\'/g" +} + +#------------------------------------------------------------------------------ +# Main. +# + +# Determine the operating mode. +if ${__SHUNIT_BUILTIN} [ $# -eq 0 -o "${1:-}" = '--' ]; then + __shunit_script=${__SHUNIT_PARENT} + __shunit_mode=${__SHUNIT_MODE_SOURCED} +else + __shunit_script=$1 + if ! ${__SHUNIT_BUILTIN} [ -r "${__shunit_script}" ]; then + _shunit_fatal "unable to read from ${__shunit_script}" + fi + __shunit_mode=${__SHUNIT_MODE_STANDALONE} +fi + +# Create a temporary storage location. +__shunit_tmpDir=`_shunit_mktempDir` + +# Provide a public temporary directory for unit test scripts. +# TODO(kward): document this. +SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" +if ! command mkdir "${SHUNIT_TMPDIR}"; then + _shunit_fatal "error creating SHUNIT_TMPDIR '${SHUNIT_TMPDIR}'" +fi + +# Configure traps to clean up after ourselves. +trap '_shunit_cleanup EXIT' 0 +trap '_shunit_cleanup INT' 2 +trap '_shunit_cleanup TERM' 15 + +# Create phantom functions to work around issues with Cygwin. +_shunit_mktempFunc +PATH="${__shunit_tmpDir}:${PATH}" + +# Make sure phantom functions are executable. This will bite if `/tmp` (or the +# current `$TMPDIR`) points to a path on a partition that was mounted with the +# 'noexec' option. The noexec command was created with `_shunit_mktempFunc()`. +noexec 2>/dev/null || _shunit_fatal \ + 'Please declare TMPDIR with path on partition with exec permission.' + +# We must manually source the tests in standalone mode. +if ${__SHUNIT_BUILTIN} [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then + # shellcheck disable=SC1090 + ${__SHUNIT_BUILTIN} . "`_shunit_prepForSourcing \"${__shunit_script}\"`" +fi + +# Configure default output coloring behavior. +_shunit_configureColor "${SHUNIT_COLOR}" + +__shunit_startSuiteTime=`${__SHUNIT_CMD_DATE_SECONDS}` + +# Execute the oneTimeSetUp function (if it exists). +if ! oneTimeSetUp; then + _shunit_fatal "oneTimeSetUp() returned non-zero return code." +fi + +# Command line selected tests or suite selected tests +if ${__SHUNIT_BUILTIN} [ "$#" -ge 2 ]; then + # Argument $1 is either the filename of tests or '--'; either way, skip it. + shift + # Remaining arguments ($2 .. $#) are assumed to be: + # - test function names. + # - configuration options, that is started with the `--` prefix. + # Interate through all remaining args in "$@" in a POSIX (likely portable) way. + # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop + for _shunit_arg_ do + case "${_shunit_arg_}" in + --output-junit-xml=*) + # It is a request for JUnit XML output. + __shunit_junitXmlOutputFile="${_shunit_arg_#--output-junit-xml=}" + ;; + --suite-name=*) + # It is a request for a custom suite name. + __shunit_suiteName="${_shunit_arg_#--suite-name=}" + ;; + --*) + _shunit_fatal "unrecognized option \"${_shunit_arg_}\"" + ;; + *) + # It is the test name, process it in a usual way. + suite_addTest "${_shunit_arg_}" + ;; + esac + done + unset _shunit_arg_ +else + # Execute the suite function defined in the parent test script. + # DEPRECATED as of 2.1.0. + suite +fi + +# If no tests or suite specified, dynamically build a list of functions. +if ${__SHUNIT_BUILTIN} [ -z "${__shunit_suite}" ]; then + shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` + for shunit_func_ in ${shunit_funcs_}; do + suite_addTest "${shunit_func_}" + done +fi +unset shunit_func_ shunit_funcs_ + +# If suite name is not defined, dynamically generate it from the script name. +if ${__SHUNIT_BUILTIN} [ -z "${__shunit_suiteName}" ]; then + __shunit_suiteName="${__shunit_script##*/}" +fi + +# Prepare the suite name for XML output. +__shunit_xmlSuiteName="`_shunit_escapeXmlData "${__shunit_suiteName}"`" + +# Execute the suite of unit tests. +_shunit_execSuite + +# Execute the oneTimeTearDown function (if it exists). +if ! oneTimeTearDown; then + _shunit_fatal "oneTimeTearDown() returned non-zero return code." +fi + +__shunit_endSuiteTime=`${__SHUNIT_CMD_DATE_SECONDS}` + +# Generate a report summary. +_shunit_generateReport + +# That's it folks. +if ! ${__SHUNIT_BUILTIN} [ "${__shunit_testsFailed}" -eq 0 ]; then + return ${SHUNIT_FALSE} +fi From 5151eacdaf46f588f330c2e45fbed7fa0a7c192e Mon Sep 17 00:00:00 2001 From: Alexander Dobrzhansky Date: Thu, 29 May 2025 18:02:07 +0200 Subject: [PATCH 018/220] Release 8.0.2 (#452) --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index d6d131576..92649b72a 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=5e347d3532ff15bb888a78d851e87cf5cc1956edd32b5d4a0cac3220da0a5a0b +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.2.tar.gz +ENV REDIS_DOWNLOAD_SHA=caf3c0069f06fc84c5153bd2a348b204c578de80490c73857bee01d9b5d7401f RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index fc46b649e..1c081b9c6 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=5e347d3532ff15bb888a78d851e87cf5cc1956edd32b5d4a0cac3220da0a5a0b +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.2.tar.gz +ENV REDIS_DOWNLOAD_SHA=caf3c0069f06fc84c5153bd2a348b204c578de80490c73857bee01d9b5d7401f RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From c0091fa744e1437a972c5ee5c8062dc1894e6ab7 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Thu, 19 Jun 2025 17:05:53 +0300 Subject: [PATCH 019/220] Redis 8.2 m01 (#462) * Update Alpine Dockerfile for Redis 8.2-m01 - Bump Alpine base image from 3.21 to 3.22 - Update Redis download URL to 8.2-m01 - Update SHA256 checksum * Update Debian Dockerfile for Redis 8.2-m01 - Update Redis download URL to 8.2-m01 - Update SHA256 checksum * Update Alpine Dockerfile build dependencies - Replace clang18-libclang with clang-static and clang-libclang - Add llvm-dev and ncurses-dev packages - Add RUST_DYN_CRT=1 export to disable static linking for RediSearch's rust submodule * Set fail-fast to false in premerge workflow This ensures all matrix jobs continue running even if some fail, providing complete feedback on which platforms/distributions work. * Run tests on ARM32/64 --- .../actions/build-and-tag-locally/action.yml | 20 +++++++++---------- .github/workflows/pre-merge.yml | 1 + alpine/Dockerfile | 13 ++++++++---- alpine/docker-entrypoint.sh | 4 ++-- debian/Dockerfile | 4 ++-- test/run-entrypoint-tests.sh | 12 +++++++++++ 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index f67023fd5..35f234b2f 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -112,25 +112,25 @@ runs: - name: Run container shell: bash - if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} run: | docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - name: Container Logs - if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker logs sanity-test-${{ steps.platform.outputs.display_name }} - name: Sanity Tests - if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server - name: Verify installed modules - if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | modules=$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli module list) @@ -150,7 +150,7 @@ runs: fi - name: Test RedisBloom - if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.ADD popular_keys "redis:hash" @@ -160,7 +160,7 @@ runs: echo "RedisBloom test passed successfully" - name: Test RediSearch - if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli FT.CREATE redis_commands ON HASH PREFIX 1 cmd: SCHEMA name TEXT SORTABLE description TEXT @@ -175,7 +175,7 @@ runs: fi - name: Test RedisTimeSeries - if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli TS.CREATE redis:cpu:usage RETENTION 86400 @@ -191,7 +191,7 @@ runs: fi - name: Test ReJSON - if: ${{ contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli JSON.SET redis:config $ '{"maxmemory":"2gb","maxmemory-policy":"allkeys-lru"}' @@ -206,7 +206,7 @@ runs: - name: Test the entrypoint id: test_entrypoint - if: ${{ contains(fromJSON('["amd64", "i386"]'), steps.platform.outputs.display_name) }} + if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash run: > cd test && env @@ -226,7 +226,7 @@ runs: - name: Push image uses: docker/build-push-action@v6 - if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64"]'), steps.platform.outputs.display_name) }} + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} with: context: ${{ inputs.distribution }} push: true diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 87585b26a..79ce49c67 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -9,6 +9,7 @@ jobs: build-and-test: runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} strategy: + fail-fast: false matrix: distribution: - debian diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 92649b72a..b3f639727 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21 +FROM alpine:3.22 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added RUN set -eux; \ @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.2.tar.gz -ENV REDIS_DOWNLOAD_SHA=caf3c0069f06fc84c5153bd2a348b204c578de80490c73857bee01d9b5d7401f +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ @@ -43,7 +43,8 @@ RUN set -eux; \ build-base \ cargo \ clang \ - clang18-libclang \ + clang-static \ + clang-libclang \ cmake \ curl \ g++ \ @@ -51,6 +52,8 @@ RUN set -eux; \ libffi-dev \ libgcc \ libtool \ + llvm-dev \ + ncurses-dev \ openssh \ openssl \ py-virtualenv \ @@ -100,6 +103,8 @@ RUN set -eux; \ sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \ grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \ \ +# Disable static linking the C runtime for RediSearch's rust submodule + export RUST_DYN_CRT=1; \ export BUILD_TLS=yes; \ if [ "$BUILD_WITH_MODULES" = "yes" ]; then \ make -C /usr/src/redis/modules/redisjson get_source; \ diff --git a/alpine/docker-entrypoint.sh b/alpine/docker-entrypoint.sh index 476913d93..ddbd1247e 100755 --- a/alpine/docker-entrypoint.sh +++ b/alpine/docker-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -SETPRIV="/usr/bin/setpriv --reuid redis --regid redis --clear-groups" +SETPRIV="/bin/setpriv --reuid redis --regid redis --clear-groups" IS_REDIS_SENTINEL="" IS_REDIS_SERVER="" CONFIG="" @@ -10,7 +10,7 @@ SKIP_FIX_PERMS_NOTICE="Use SKIP_FIX_PERMS=1 to skip permission changes." # functions has_cap() { - /usr/bin/setpriv -d | grep -q 'Capability bounding set:.*\b'"$1"'\b' + /bin/setpriv -d | grep -q 'Capability bounding set:.*\b'"$1"'\b' } check_for_sentinel() { diff --git a/debian/Dockerfile b/debian/Dockerfile index 1c081b9c6..ab55b7642 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.0.2.tar.gz -ENV REDIS_DOWNLOAD_SHA=caf3c0069f06fc84c5153bd2a348b204c578de80490c73857bee01d9b5d7401f +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz +ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 500fc3cec..28d138ae0 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -20,6 +20,9 @@ # ## +# Container initialization wait time in seconds +CONTAINER_INIT_WAIT=3 + if [ -z "$REDIS_IMG" ]; then echo "REDIS_IMG may not be empty" exit 1 @@ -38,6 +41,7 @@ get_container_user_uid_gid_on_the_host() { container_user="$1" dir=$(mktemp -d -p .) docker run --rm -v "$(pwd)/$dir":/w -w /w --entrypoint=/bin/sh "$REDIS_IMG" -c "chown $container_user ." + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize stat -c "%u %g" "$dir" sudo rm -rf "$dir" } @@ -177,6 +181,7 @@ run_docker_and_test_ownership() { fi docker_output=$($docker_run 2>&1) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize if [ "$TEST_VERBOSE" ]; then echo "After:" @@ -265,6 +270,7 @@ run_redis_docker_and_check_uid_gid() { docker_cmd="$*" # shellcheck disable=SC2086 container=$(docker run $docker_flags -d "$REDIS_IMG" $docker_cmd) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize ret=$? assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" @@ -296,6 +302,7 @@ run_redis_docker_and_check_modules() { docker_cmd="$1" # shellcheck disable=SC2086 container=$(docker run --rm -d "$REDIS_IMG" $docker_cmd) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize info=$(docker exec "$container" redis-cli info) [ "$PLATFORM" ] && [ "$PLATFORM" != "amd64" ] && startSkipping @@ -322,6 +329,7 @@ assert_redis_v8() { test_redis_version() { ret=$(docker run --rm "$REDIS_IMG" -v|tail -n 1) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize assert_redis_v8 "$ret" } @@ -545,6 +553,7 @@ test_redis_server_persistence_with_bind_mount() { chmod 0444 "$dir" container=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG" --appendonly yes) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -559,6 +568,7 @@ test_redis_server_persistence_with_bind_mount() { sudo chown -R "$HOST_OWNER" "$dir" container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -576,6 +586,7 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data' container=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG" --appendonly yes) + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -590,6 +601,7 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data && chmod 0000 -R /data' container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") + sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" From 0420644d05b5d9e1a7a041d9d8ad471ff910a757 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Thu, 3 Jul 2025 08:05:59 +0300 Subject: [PATCH 020/220] Add push trigger for release/8.2 and update Redis version to 8.2-rc1-int - Add push trigger for release/8.2 branch in GitHub Actions workflow - Update Redis version from 8.2-m01 to 8.2-rc1-int in both Alpine and Debian Dockerfiles - Update SHA256 hash for the new Redis version tag --- .github/workflows/pre-merge.yml | 3 +++ alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 79ce49c67..2b0a81525 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -4,6 +4,9 @@ on: branches: - master - release/* + push: + branches: + - release/8.2 jobs: build-and-test: diff --git a/alpine/Dockerfile b/alpine/Dockerfile index b3f639727..df70d75c6 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=afb487aa209aec5d607f6e6a3e03d97946933ddcd24e2069d24f76b47e668571 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index ab55b7642..8bfc79a88 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=afb487aa209aec5d607f6e6a3e03d97946933ddcd24e2069d24f76b47e668571 RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From dc9ceb8661ea4bfc660d73d6889533e0ae8fb3f3 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Sat, 5 Jul 2025 15:50:17 +0300 Subject: [PATCH 021/220] Redis 8.2-rc1 (#465) --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index b3f639727..fffe2f9ae 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1.tar.gz +ENV REDIS_DOWNLOAD_SHA=563adfc7dc5384a05cdf1ee2059cfe4ad50ee311b67b5793ecf7a915ef4a27b9 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index ab55b7642..3868cae98 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-m01.tar.gz -ENV REDIS_DOWNLOAD_SHA=0a1d5a27dd9cb13d51bcaba6546dd54b529774ede5d8993fce5c3ac2bace0f72 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1.tar.gz +ENV REDIS_DOWNLOAD_SHA=563adfc7dc5384a05cdf1ee2059cfe4ad50ee311b67b5793ecf7a915ef4a27b9 RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From e2ff5f7300b3151c66af3d2b7f2e5c362b2d216c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 16:02:05 +0300 Subject: [PATCH 022/220] release 8.2-int (8.2 GA internal) --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index df70d75c6..0dff2a7d1 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=afb487aa209aec5d607f6e6a3e03d97946933ddcd24e2069d24f76b47e668571 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=8ca229b34a2ebf6a5355edf2fa9c6fcd2233823a39a063baa6b16df12bb366af RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 8bfc79a88..87f028187 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-rc1-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=afb487aa209aec5d607f6e6a3e03d97946933ddcd24e2069d24f76b47e668571 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=8ca229b34a2ebf6a5355edf2fa9c6fcd2233823a39a063baa6b16df12bb366af RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 5459e303572e1a37b078712ab04dcf390784bf27 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Mon, 4 Aug 2025 14:45:37 +0300 Subject: [PATCH 023/220] Release 8.2.0 --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index 0dff2a7d1..abe94fcce 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=8ca229b34a2ebf6a5355edf2fa9c6fcd2233823a39a063baa6b16df12bb366af +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 87f028187..ad615c384 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=8ca229b34a2ebf6a5355edf2fa9c6fcd2233823a39a063baa6b16df12bb366af +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From a13b78815d980881e57f15b9cf13cd2f26f3fab6 Mon Sep 17 00:00:00 2001 From: adamiBs Date: Mon, 18 Aug 2025 19:44:26 +0300 Subject: [PATCH 024/220] Bump Redis version to 8.2.1 (#469) - Update REDIS_DOWNLOAD_URL to use official Redis 8.2.1 release - Update REDIS_DOWNLOAD_SHA to e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index abe94fcce..e34634014 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed +ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index ad615c384..0e7d71718 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed +ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From bb9ce957802a20224a0793f7a0a895b13d5a96fa Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 13:15:37 +0300 Subject: [PATCH 025/220] First branch manipulation implementation --- .../actions/ensure-release-branch/action.yml | 12 +++ .../ensure-release-branch.sh | 84 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 16 ++++ 3 files changed, 112 insertions(+) create mode 100644 .github/actions/ensure-release-branch/action.yml create mode 100755 .github/actions/ensure-release-branch/ensure-release-branch.sh create mode 100644 .github/workflows/release_build_and_test.yml diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml new file mode 100644 index 000000000..52ab05d4a --- /dev/null +++ b/.github/actions/ensure-release-branch/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh new file mode 100755 index 000000000..465e20f25 --- /dev/null +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -e + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Configure Git to use GITHUB_TOKEN for authentication +if [ -n "$GITHUB_TOKEN" ]; then + echo "Configuring Git with GITHUB_TOKEN..." + git config --global url."https://x-access-token:$%7BGITHUB_TOKEN%7D@github.com/".insteadOf "https://github.com/" + + # Set Git user for commits (required for GitHub Actions) + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" +else + echo "Warning: GITHUB_TOKEN not found. Git operations may fail if authentication is required." +fi + +# Define RELEASE_VERSION_BRANCH which is the same as TAG +RELEASE_VERSION_BRANCH="$TAG" + +echo "TAG: $TAG" +echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" + +# Check if RELEASE_VERSION_BRANCH exists in origin +if git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" | grep -q "$RELEASE_VERSION_BRANCH"; then + echo "Branch $RELEASE_VERSION_BRANCH exists in origin, checking out..." + git fetch origin "$RELEASE_VERSION_BRANCH" + git checkout "$RELEASE_VERSION_BRANCH" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + exit 0 +fi + +echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" + +# Detect RELEASE_BRANCH name (release/X.Y format) +RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" +echo "RELEASE_BRANCH: $RELEASE_BRANCH" + +# Check if RELEASE_BRANCH exists in origin +if git ls-remote --heads origin "$RELEASE_BRANCH" | grep -q "$RELEASE_BRANCH"; then + echo "Branch $RELEASE_BRANCH exists in origin" + git fetch origin "$RELEASE_BRANCH" + git checkout "$RELEASE_BRANCH" +else + echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" + + # Detect base branch (previous existing branch for the version) + MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') + MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) + MINOR=$(echo "$MAJOR_MINOR" | cut -d. -f2) + + # Find the previous existing release branch + BASE_BRANCH=$(git ls-remote --heads origin "release/$MAJOR.[0-9]" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) + echo git ls-remote --heads origin "release/$MAJOR.[0-9]" + + if [ -z "$BASE_BRANCH" ]; then + echo "Error: Could not find a base branch for $RELEASE_BRANCH" + exit 1 + fi + + echo "Using base branch: $BASE_BRANCH" + + # Create new branch based on base branch and push to origin + git fetch origin "$BASE_BRANCH" + git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + git push origin "$RELEASE_BRANCH" + echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" +fi + +# At this point, we should be on RELEASE_BRANCH +echo "Current branch: $(git branch --show-current)" + +# Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin +git checkout -b "$RELEASE_VERSION_BRANCH" +git push origin "$RELEASE_VERSION_BRANCH" +echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" + +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml new file mode 100644 index 000000000..6a7edd108 --- /dev/null +++ b/.github/workflows/release_build_and_test.yml @@ -0,0 +1,16 @@ +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to build' + required: true + +jobs: + build-and-test: + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Ensure Release Branch + uses: ./.github/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From bced1dabf834b49ac7a89d418cf0e396a1de3171 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 13:27:40 +0300 Subject: [PATCH 026/220] Fix workflow --- .github/workflows/release_build_and_test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6a7edd108..b0a1853e8 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,12 +1,13 @@ on: workflow_dispatch: - inputs: - release_tag: - description: 'Release tag to build' - required: true + inputs: + release_tag: + description: 'Release tag to build' + required: true jobs: build-and-test: + runs-on: ["self-hosted"] steps: - name: Checkout code uses: actions/checkout@v4 From 36d63fc32680ad63990fab4ef9e5728f60095644 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 14:22:57 +0300 Subject: [PATCH 027/220] Fix yaml --- .github/workflows/release_build_and_test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b0a1853e8..6f7009e03 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -7,11 +7,11 @@ on: jobs: build-and-test: - runs-on: ["self-hosted"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Ensure Release Branch - uses: ./.github/ensure-release-branch - with: - release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file + runs-on: ["self-hosted"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Ensure Release Branch + uses: ./.github/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From 6d12d22e06488a7ca2ec725bd384267c8aa5561c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 14:25:30 +0300 Subject: [PATCH 028/220] Fix path to actions --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6f7009e03..6856b8460 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -12,6 +12,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure Release Branch - uses: ./.github/ensure-release-branch + uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From d067295e4a9fa195ac3f3625ebaeb0fa1cfd3569 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 17:11:09 +0300 Subject: [PATCH 029/220] Fix push empty branches --- .../actions/ensure-release-branch/ensure-release-branch.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 465e20f25..973108741 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +#set -x # Input TAG is expected in $1 TAG="$1" @@ -69,7 +70,7 @@ else # Create new branch based on base branch and push to origin git fetch origin "$BASE_BRANCH" git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - git push origin "$RELEASE_BRANCH" + git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi @@ -78,7 +79,7 @@ echo "Current branch: $(git branch --show-current)" # Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin git checkout -b "$RELEASE_VERSION_BRANCH" -git push origin "$RELEASE_VERSION_BRANCH" +git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From b82ea9d86eb182865b814131aeb0e9d6d63bc474 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 17:18:36 +0300 Subject: [PATCH 030/220] Use different runner --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6856b8460..9febc219f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -7,7 +7,7 @@ on: jobs: build-and-test: - runs-on: ["self-hosted"] + runs-on: ["ubuntu-latest"] steps: - name: Checkout code uses: actions/checkout@v4 From 32823d7cfcba567423c52b5866848da1215ae4a7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 20:30:55 +0300 Subject: [PATCH 031/220] Pretty output --- .../ensure-release-branch.sh | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 973108741..cd14e32c7 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -2,6 +2,19 @@ set -e #set -x +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/helpers.sh" + # Input TAG is expected in $1 TAG="$1" @@ -9,19 +22,6 @@ if [ -z "$TAG" ]; then echo "Error: TAG is required as first argument" exit 1 fi - -# Configure Git to use GITHUB_TOKEN for authentication -if [ -n "$GITHUB_TOKEN" ]; then - echo "Configuring Git with GITHUB_TOKEN..." - git config --global url."https://x-access-token:$%7BGITHUB_TOKEN%7D@github.com/".insteadOf "https://github.com/" - - # Set Git user for commits (required for GitHub Actions) - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" -else - echo "Warning: GITHUB_TOKEN not found. Git operations may fail if authentication is required." -fi - # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" @@ -29,10 +29,10 @@ echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" # Check if RELEASE_VERSION_BRANCH exists in origin -if git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" | grep -q "$RELEASE_VERSION_BRANCH"; then - echo "Branch $RELEASE_VERSION_BRANCH exists in origin, checking out..." - git fetch origin "$RELEASE_VERSION_BRANCH" - git checkout "$RELEASE_VERSION_BRANCH" +execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then + execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" exit 0 fi @@ -44,21 +44,21 @@ RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" # Check if RELEASE_BRANCH exists in origin -if git ls-remote --heads origin "$RELEASE_BRANCH" | grep -q "$RELEASE_BRANCH"; then +execute_command git ls-remote --heads origin "$RELEASE_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - git fetch origin "$RELEASE_BRANCH" - git checkout "$RELEASE_BRANCH" + execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git checkout "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" # Detect base branch (previous existing branch for the version) MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) - MINOR=$(echo "$MAJOR_MINOR" | cut -d. -f2) # Find the previous existing release branch - BASE_BRANCH=$(git ls-remote --heads origin "release/$MAJOR.[0-9]" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) - echo git ls-remote --heads origin "release/$MAJOR.[0-9]" + execute_command git ls-remote --heads origin "release/$MAJOR.[0-9]" + BASE_BRANCH=$(echo "$last_cmd_stdout" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) if [ -z "$BASE_BRANCH" ]; then echo "Error: Could not find a base branch for $RELEASE_BRANCH" @@ -68,9 +68,9 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - git fetch origin "$BASE_BRANCH" - git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - git push origin HEAD:"$RELEASE_BRANCH" + execute command git fetch origin "$BASE_BRANCH" + execute command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + execute command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi @@ -78,8 +78,8 @@ fi echo "Current branch: $(git branch --show-current)" # Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin -git checkout -b "$RELEASE_VERSION_BRANCH" -git push origin HEAD:"$RELEASE_VERSION_BRANCH" +execute_command git checkout -b "$RELEASE_VERSION_BRANCH" +execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From efe9b17fe14c77572087b096af11964de57150e0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 20:32:19 +0300 Subject: [PATCH 032/220] Fix execut command --- .../actions/ensure-release-branch/ensure-release-branch.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index cd14e32c7..9e7bd5ab1 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,9 +68,9 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute command git fetch origin "$BASE_BRANCH" - execute command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - execute command git push origin HEAD:"$RELEASE_BRANCH" + execute_command git fetch origin "$BASE_BRANCH" + execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi From 68817c7f9a152e001064664b0946d5ef85e5b27b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 12:20:16 +0300 Subject: [PATCH 033/220] Use common base --- .github/actions/common/helpers.sh | 83 +++++++++++++++++++ .../ensure-release-branch.sh | 8 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 .github/actions/common/helpers.sh diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh new file mode 100644 index 000000000..3a74f3062 --- /dev/null +++ b/.github/actions/common/helpers.sh @@ -0,0 +1,83 @@ +# Function to execute command from array and capture output +execute_command() { +# turn off errexit (set -e) if it is active and restore it later + echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" + + local cmd + + # Check if no arguments provided + if [ $# -eq 0 ]; then + # Check if cmd_array variable exists and is an array + if declare -p cmd_array 2>/dev/null | grep -q "declare -a"; then + # Use the existing cmd_array variable + cmd=("${cmd_array[@]}") + else + echo "Error: No arguments provided and cmd_array variable not found or not an array" >&2 + return 1 + fi + else + cmd=("$@") + fi + + # Create temporary files for stdout and stderr + local stdout_file stderr_file + stdout_file=$(mktemp) + stderr_file=$(mktemp) + + # Execute command and capture output + console_output 1 gray "Executing command: ${cmd[*]}" + "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" + last_cmd_result=$? + + # Read captured output + last_cmd_stdout=$(cat "$stdout_file") + last_cmd_stderr=$(cat "$stderr_file") + + if [ "$last_cmd_result" -ne 0 ]; then + console_output 0 red "Command failed with exit code $last_cmd_result" + console_output 0 red "Standard Output:" + console_output 0 red "$last_cmd_stdout" + console_output 0 red "Standard Error:" + console_output 0 red "$last_cmd_stderr" + fi + + # Clean up temporary files + rm -f "$stdout_file" "$stderr_file" + + [ "$restore_errexit" ] && set -e + return $last_cmd_result +} + +# Helper function to output multiline variables with color +console_output() { + local verbosity_level="$1" + local color="$2" + local content="$3" + local current_verbosity="${VERBOSITY:-0}" + + # Check if we should output based on verbosity level + if [ "$current_verbosity" -ge "$verbosity_level" ]; then + local color_code="" + local reset_code="\033[0m" + + case "$color" in + "gray"|"grey") + color_code="\033[90m" + ;; + "white") + color_code="\033[97m" + ;; + "red") + color_code="\033[91m" + ;; + *) + color_code="\033[0m" # Default to no color + ;; + esac + + # Output each line with 4-space indent and color + while IFS= read -r line || [ -n "$line" ]; do + printf "${color_code} %s${reset_code}\n" "$line" + done <<< "$content" + fi +} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 9e7bd5ab1..e3baf7673 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,4 +1,10 @@ #!/bin/bash + +# This script ensures that a release branch and release version branch exist for a given release tag. +# It creates and pushes both branches if they do not exist. +# It also checks out the release version branch at the end. +# https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation + set -e #set -x @@ -13,7 +19,7 @@ VERBOSITY=1 SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$SCRIPT_DIR/helpers.sh" +. "$SCRIPT_DIR/../common/helpers.sh" # Input TAG is expected in $1 TAG="$1" From 3041ccdb126d42e65587bcfbcb502eb7ba1f33b9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 12:57:15 +0300 Subject: [PATCH 034/220] Add validate archive action --- .../validate-redis-release-archive/action.yml | 12 +++++ .../validate-redis-release-archive.sh | 47 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 5 ++ 3 files changed, 64 insertions(+) create mode 100644 .github/actions/validate-redis-release-archive/action.yml create mode 100644 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/validate-redis-release-archive/action.yml b/.github/actions/validate-redis-release-archive/action.yml new file mode 100644 index 000000000..fcd775ce7 --- /dev/null +++ b/.github/actions/validate-redis-release-archive/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/validate-redis-release-archive.sh ${{ inputs.release_tag }} \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh new file mode 100644 index 000000000..8d038cade --- /dev/null +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# This script validates a Redis release archive by downloading it and calculating its SHA256 sum. +# It constructs the URL, downloads the file, calculates the hash, and exports environment variables. + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Construct Redis archive URL +REDIS_ARCHIVE_URL="https://github.com/redis/redis/archive/refs/tags/$%7BTAG%7D.tar.gz" +echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" + +# Download the Redis archive +TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" +echo "Downloading Redis archive to $TEMP_ARCHIVE..." +if ! curl -L -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then + echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" + exit 1 +fi + +# Calculate SHA256 sum +echo "Calculating SHA256 sum..." +REDIS_ARCHIVE_SHA=$(sha256sum "$TEMP_ARCHIVE" | cut -d' ' -f1) +echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" + +# Write variables to GITHUB_ENV +if [ -n "$GITHUB_ENV" ]; then + echo "REDIS_ARCHIVE_URL=$REDIS_ARCHIVE_URL" >> "$GITHUB_ENV" + echo "REDIS_ARCHIVE_SHA=$REDIS_ARCHIVE_SHA" >> "$GITHUB_ENV" + echo "Environment variables written to $GITHUB_ENV" +else + echo "Error: GITHUB_ENV not set" + # Clean up temporary file + rm -f "$TEMP_ARCHIVE" + exit 1 +fi + +# Clean up temporary file +rm -f "$TEMP_ARCHIVE" + +echo "Redis archive validation completed successfully" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 9febc219f..7451198c4 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -11,6 +11,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Validate Redis Release Archive + uses: ./.github/actions/validate-redis-release-archive + with: + release_tag: ${{ github.event.inputs.release_tag }} + - name: Ensure Release Branch uses: ./.github/actions/ensure-release-branch with: From 5637a313a6e415d064fec19dc137d7da9c87d16d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:29:50 +0300 Subject: [PATCH 035/220] Make executable --- .../validate-redis-release-archive.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh old mode 100644 new mode 100755 From c17467a24300e09bd84af4c8d20d54c2cbda5b85 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:32:35 +0300 Subject: [PATCH 036/220] Make curl return error --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index 8d038cade..be450a4f6 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -L -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -fiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From 1d007596a833f80cdb0a11215c138ceaa700d8a4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:33:33 +0300 Subject: [PATCH 037/220] Make curl silent --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index be450a4f6..d258369aa 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -fiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -sfiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From a86989c8bc6a4762c2247511cbe6743fd488a40c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:25:39 +0300 Subject: [PATCH 038/220] Try to use pre-merge.yml --- .../ensure-release-branch.sh | 42 ++++++++++++++++--- .github/workflows/release_build_and_test.yml | 8 +++- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index e3baf7673..037248e92 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,12 +1,12 @@ #!/bin/bash +set -e +#set -x # This script ensures that a release branch and release version branch exist for a given release tag. # It creates and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation -set -e -#set -x # shellcheck disable=SC2034 last_cmd_stdout="" @@ -17,17 +17,43 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 + SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" -# Input TAG is expected in $1 -TAG="$1" +# Parse arguments +CREATE="" +TAG="" + +while [[ $# -gt 0 ]]; do + case $1 in + --create) + CREATE=1 + shift + ;; + -*) + echo "Error: Unknown option $1" + exit 1 + ;; + *) + if [ -z "$TAG" ]; then + TAG="$1" + else + echo "Error: Multiple TAG arguments provided" + exit 1 + fi + shift + ;; + esac +done if [ -z "$TAG" ]; then - echo "Error: TAG is required as first argument" + echo "Error: TAG is required as argument" + echo "Usage: $0 [--no-write] " exit 1 fi + # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" @@ -44,6 +70,10 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" +if [ -z "$CREATE" ]; then + echo "Refuse to modify repository without --create option" + exit 1 +fi # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" @@ -88,4 +118,4 @@ execute_command git checkout -b "$RELEASE_VERSION_BRANCH" execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 7451198c4..72f28679c 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -11,6 +11,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Validate Redis Release Archive uses: ./.github/actions/validate-redis-release-archive with: @@ -19,4 +20,9 @@ jobs: - name: Ensure Release Branch uses: ./.github/actions/ensure-release-branch with: - release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file + release_tag: ${{ github.event.inputs.release_tag }} + + - name: Apply Docker Version + uses: ./.github/actions/apply-docker-version + with: + release_tag: ${{ github.event.inputs.release_tag }} From 5f6fccc7234ab72f1e3e958f418c2515a4295049 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:27:38 +0300 Subject: [PATCH 039/220] allow-modify flag --- .github/actions/ensure-release-branch/action.yml | 5 ++++- .../ensure-release-branch/ensure-release-branch.sh | 14 +++++++------- .github/workflows/pre-merge.yml | 13 ++++++++++--- .github/workflows/release_build_and_test.yml | 11 ++++++++++- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 52ab05d4a..8426db2a8 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -2,6 +2,9 @@ inputs: release_tag: description: 'Release tag to build' required: true + allow_modify: + description: 'Allow modifying the repository' + default: false runs: using: "composite" @@ -9,4 +12,4 @@ runs: - name: Ensure Release Branch shell: bash run: | - ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.release_tag }} + ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 037248e92..9a41004cf 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -3,7 +3,7 @@ set -e #set -x # This script ensures that a release branch and release version branch exist for a given release tag. -# It creates and pushes both branches if they do not exist. +# It allow-modifys and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation @@ -23,13 +23,13 @@ SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" . "$SCRIPT_DIR/../common/helpers.sh" # Parse arguments -CREATE="" +ALLOW_MODIFY="" TAG="" while [[ $# -gt 0 ]]; do case $1 in - --create) - CREATE=1 + --allow-modify) + ALLOW_MODIFY=1 shift ;; -*) @@ -50,7 +50,7 @@ done if [ -z "$TAG" ]; then echo "Error: TAG is required as argument" - echo "Usage: $0 [--no-write] " + echo "Usage: $0 [--allow-modify] " exit 1 fi @@ -70,8 +70,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$CREATE" ]; then - echo "Refuse to modify repository without --create option" +if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" exit 1 fi diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 2b0a81525..7cb53b149 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -4,9 +4,11 @@ on: branches: - master - release/* - push: - branches: - - release/8.2 + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to build' + required: true jobs: build-and-test: @@ -40,6 +42,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Ensure release branch + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: ./.github/actions/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 72f28679c..24cae9095 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -6,7 +6,7 @@ on: required: true jobs: - build-and-test: + prepare-release: runs-on: ["ubuntu-latest"] steps: - name: Checkout code @@ -21,8 +21,17 @@ jobs: uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: true - name: Apply Docker Version uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} + + build-and-test: + needs: prepare-release + runs-on: ["ubuntu-latest"] + use: pre-merge.yml + with: + release_tag: ${{ github.event.inputs.release_tag }} + From 8e1e9ad126b0eee82782c85e11db2288cf0e7359 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:28:56 +0300 Subject: [PATCH 040/220] Fix workflow --- .github/workflows/release_build_and_test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 24cae9095..5fbf9f46d 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -28,10 +28,10 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} - build-and-test: - needs: prepare-release - runs-on: ["ubuntu-latest"] - use: pre-merge.yml - with: - release_tag: ${{ github.event.inputs.release_tag }} + build-and-test: + needs: prepare-release + runs-on: ["ubuntu-latest"] + use: pre-merge.yml + with: + release_tag: ${{ github.event.inputs.release_tag }} From b981b62dd78cdb4f8504cc70a939b032789811e0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:31:34 +0300 Subject: [PATCH 041/220] Fix worflow reuse --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 7cb53b149..fe2734330 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -4,7 +4,7 @@ on: branches: - master - release/* - workflow_dispatch: + workflow_call: inputs: release_tag: description: 'Release tag to build' @@ -43,7 +43,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure release branch - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_call' }} uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5fbf9f46d..ee620a802 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -29,9 +29,10 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: - needs: prepare-release runs-on: ["ubuntu-latest"] - use: pre-merge.yml + needs: prepare-release + uses: ./.github/workflows/pre-merge.yml + secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 07869702053c4eaef3d1ab65605c9a4879df6993 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:34:07 +0300 Subject: [PATCH 042/220] Fixed required property type --- .github/workflows/pre-merge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index fe2734330..51a50a3a2 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -9,6 +9,7 @@ on: release_tag: description: 'Release tag to build' required: true + type: string jobs: build-and-test: From d7507d3b88eff8e5d65f8cfbc91065449edff95c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:38:55 +0300 Subject: [PATCH 043/220] Remove runs-on --- .github/workflows/release_build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ee620a802..10ea16c37 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -29,7 +29,6 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: - runs-on: ["ubuntu-latest"] needs: prepare-release uses: ./.github/workflows/pre-merge.yml secrets: inherit From d3d5043e6b9b3224c48f04d17f9d39c7c7ad2a6e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:39:37 +0300 Subject: [PATCH 044/220] Add apply-docker-version --- .../actions/apply-docker-version/action.yml | 12 +++ .../apply-docker-version.sh | 88 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 .github/actions/apply-docker-version/action.yml create mode 100755 .github/actions/apply-docker-version/apply-docker-version.sh diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml new file mode 100644 index 000000000..fd02d56ba --- /dev/null +++ b/.github/actions/apply-docker-version/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh new file mode 100755 index 000000000..6fe9e3eef --- /dev/null +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +# This script updates Redis version in Dockerfiles using environment variables +# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Check if required environment variables are set +if [ -z "$REDIS_ARCHIVE_URL" ]; then + echo "Error: REDIS_ARCHIVE_URL environment variable is not set" + exit 1 +fi + +if [ -z "$REDIS_ARCHIVE_SHA" ]; then + echo "Error: REDIS_ARCHIVE_SHA environment variable is not set" + exit 1 +fi + +echo "TAG: $TAG" +echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" +echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" + +# Function to update Dockerfile +update_dockerfile() { + local dockerfile="$1" + local updated=false + + if [ ! -f "$dockerfile" ]; then + echo "Warning: $dockerfile not found, skipping" + return 1 + fi + + echo "Updating $dockerfile..." + + # Update REDIS_DOWNLOAD_URL + if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile" + updated=true + echo " Updated REDIS_DOWNLOAD_URL" + fi + + # Update REDIS_DOWNLOAD_SHA + if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile" + updated=true + echo " Updated REDIS_DOWNLOAD_SHA" + fi + + if [ "$updated" = true ]; then + echo " $dockerfile updated successfully" + return 0 + else + echo " No changes needed in $dockerfile" + return 1 + fi +} + +# Track if any files were modified +files_modified=false + +# Update debian/Dockerfile +if update_dockerfile "debian/Dockerfile"; then + files_modified=true +fi + +# Update alpine/Dockerfile +if update_dockerfile "alpine/Dockerfile"; then + files_modified=true +fi + +# Commit changes if any files were modified +if [ "$files_modified" = true ]; then + echo "Files were modified, committing changes..." + git add debian/Dockerfile alpine/Dockerfile + git commit -m "$TAG" + echo "Changes committed with message: $TAG" +else + echo "No files were modified, nothing to commit" +fi + +echo "Docker version update completed" From 21faa0e128d89dee74bab8dc0cebf33d6e0e1705 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 17:18:36 +0300 Subject: [PATCH 045/220] Use explicit branch name --- .github/workflows/release_build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 10ea16c37..5ea610233 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,24 +13,24 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive + uses: ./.github/actions/validate-redis-release-archive@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch - uses: ./.github/actions/ensure-release-branch + uses: ./.github/actions/ensure-release-branch@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true - name: Apply Docker Version - uses: ./.github/actions/apply-docker-version + uses: ./.github/actions/apply-docker-version@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml + uses: ./.github/workflows/pre-merge.yml@Peter-Sh_release_automation_RED-165688 secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 880577793873b94c385192f1ff97364974c91eaa Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 17:52:05 +0300 Subject: [PATCH 046/220] Branches doesn't work with local versions --- .github/workflows/release_build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5ea610233..10ea16c37 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,24 +13,24 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/validate-redis-release-archive with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch - uses: ./.github/actions/ensure-release-branch@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true - name: Apply Docker Version - uses: ./.github/actions/apply-docker-version@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml@Peter-Sh_release_automation_RED-165688 + uses: ./.github/workflows/pre-merge.yml secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 3a85e67886a63733a39dcce7bc2b281d96f3f448 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 18:00:41 +0300 Subject: [PATCH 047/220] Add bot name --- .github/actions/apply-docker-version/apply-docker-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6fe9e3eef..6f973f077 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,6 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." + git config --global user.email "relesase-bot@redis.com" + git config --global user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git commit -m "$TAG" echo "Changes committed with message: $TAG" From 2262399b779a1a3e95ed058ca4c10a304e026a3c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 18:02:32 +0300 Subject: [PATCH 048/220] Try locl config --- .github/actions/apply-docker-version/apply-docker-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6f973f077..ae15cb93a 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,8 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config --global user.email "relesase-bot@redis.com" - git config --global user.name "Release Bot" + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git commit -m "$TAG" echo "Changes committed with message: $TAG" From cc1dd2adba92fab8d2c0de73dbd6ad74c2fa6e61 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:26:18 +0300 Subject: [PATCH 049/220] Use --author --- .github/actions/apply-docker-version/apply-docker-version.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index ae15cb93a..470fb97c4 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,10 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git commit -m "$TAG" + git commit -m "$TAG" --author "Redis Release Bot " echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" From 0a7159a9b7c08743fa5c19522bad224a2fccb33d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:33:44 +0300 Subject: [PATCH 050/220] debug output --- .github/actions/apply-docker-version/apply-docker-version.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 470fb97c4..e8bae42c0 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,8 +78,11 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git commit -m "$TAG" --author "Redis Release Bot " + git config --list --show-origin + git commit -m "$TAG" echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" From d158d12a7913821d0516fe1d8543d06d9a8fb7bd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:54:30 +0300 Subject: [PATCH 051/220] Auto merge from release branch --- .../apply-docker-version.sh | 2 +- .../ensure-release-branch.sh | 81 ++++++++++++++----- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index e8bae42c0..7bbee17c3 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -81,7 +81,7 @@ if [ "$files_modified" = true ]; then git config user.email "relesase-bot@redis.com" git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git config --list --show-origin + git diff --cached git commit -m "$TAG" echo "Changes committed with message: $TAG" else diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 9a41004cf..baa0668a5 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -e -#set -x # This script ensures that a release branch and release version branch exist for a given release tag. # It allow-modifys and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation +set -e +#set -x # shellcheck disable=SC2034 last_cmd_stdout="" @@ -60,21 +60,6 @@ RELEASE_VERSION_BRANCH="$TAG" echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" -# Check if RELEASE_VERSION_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "$RELEASE_VERSION_BRANCH" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" - exit 0 -fi - -echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 -fi - # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" @@ -84,9 +69,12 @@ execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" execute_command git fetch origin "$RELEASE_BRANCH" - execute_command git checkout "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" + if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" + exit 1 + fi # Detect base branch (previous existing branch for the version) MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') @@ -110,6 +98,61 @@ else echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi +# Check if RELEASE_VERSION_BRANCH exists in origin +execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then + execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "$RELEASE_VERSION_BRANCH" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + + # Check if there are changes in release branch that are not in release version branch + echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." + execute_command git fetch origin "$RELEASE_BRANCH" + + # Compare the two branches to see if there are commits in release branch not in release version branch + execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') + + if [ "$COMMITS_BEHIND" -gt 0 ]; then + echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + + if [ -z "$ALLOW_MODIFY" ]; then + echo "Changes detected but refusing to merge without --allow-modify option" + exit 1 + fi + + echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." + + # Try to merge the release branch into the current release version branch + execute_command git merge "origin/$RELEASE_BRANCH" --no-edit + if [ $last_cmd_result -eq 0 ]; then + echo "Successfully merged changes from $RELEASE_BRANCH" + + # Push the merged changes to origin + execute_command git push origin "$RELEASE_VERSION_BRANCH" + if [ $last_cmd_result -eq 0 ]; then + echo "Successfully pushed merged changes to origin/$RELEASE_VERSION_BRANCH" + else + echo "Error: Failed to push merged changes to origin" + exit 1 + fi + else + echo "Error: Merge failed. There may be conflicts that need manual resolution." + exit 1 + fi + fi + + exit 0 + +fi + +echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" +if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" + exit 1 +fi + +execute_command git checkout "$RELEASE_BRANCH" # At this point, we should be on RELEASE_BRANCH echo "Current branch: $(git branch --show-current)" @@ -118,4 +161,4 @@ execute_command git checkout -b "$RELEASE_VERSION_BRANCH" execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From a75454ead3bc9b1a54fab03be2ef60c72d855865 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:59:41 +0300 Subject: [PATCH 052/220] Push after merge, use helpers --- .../apply-docker-version.sh | 18 +++++++++++++++++- .../ensure-release-branch.sh | 18 ++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 7bbee17c3..6c05391d9 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -4,6 +4,21 @@ set -e # This script updates Redis version in Dockerfiles using environment variables # REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/helpers.sh" + + # Input TAG is expected in $1 TAG="$1" @@ -82,7 +97,8 @@ if [ "$files_modified" = true ]; then git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git diff --cached - git commit -m "$TAG" + execute_command git commit -m "$TAG" + execute_command git push origin "$TAG" echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index baa0668a5..47339864a 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -122,24 +122,10 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." - # Try to merge the release branch into the current release version branch execute_command git merge "origin/$RELEASE_BRANCH" --no-edit - if [ $last_cmd_result -eq 0 ]; then - echo "Successfully merged changes from $RELEASE_BRANCH" - - # Push the merged changes to origin - execute_command git push origin "$RELEASE_VERSION_BRANCH" - if [ $last_cmd_result -eq 0 ]; then - echo "Successfully pushed merged changes to origin/$RELEASE_VERSION_BRANCH" - else - echo "Error: Failed to push merged changes to origin" - exit 1 - fi - else - echo "Error: Merge failed. There may be conflicts that need manual resolution." - exit 1 - fi + # Push the merged changes to origin + execute_command git push origin "$RELEASE_VERSION_BRANCH" fi exit 0 From f7699664a5827043affb47e944ae0fa7e415dc73 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:09:06 +0300 Subject: [PATCH 053/220] debug output --- .../ensure-release-branch.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 47339864a..daeabd64c 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -123,6 +123,21 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." # Try to merge the release branch into the current release version branch + echo origin/$RELEASE_BRANCH + git log --graph --decorate -5 origin/$RELEASE_BRANCH + + echo origin/$RELEASE_VERSION_BRANCH + git log --graph --decorate -5 origin/$RELEASE_VERSION_BRANCH + + echo $RELEASE_BRANCH + git log --graph --decorate -5 $RELEASE_BRANCH + + echo $RELEASE_VERSION_BRANCH + git log --graph --decorate -5 $RELEASE_VERSION_BRANCH + + echo "HEAD" + git log --graph --decorate -5 + execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" From 9be223f751467a4cecc16c687bef3e7065161e5a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:22:16 +0300 Subject: [PATCH 054/220] Try to use --unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index daeabd64c..4bf556fb3 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,7 +68,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +92,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch origin "$BASE_BRANCH" + execute_command git fetch --unshallow origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +101,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From 49e31123419a438a5f4df8fb9a872a22eb091c6e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:29:28 +0300 Subject: [PATCH 055/220] Try to combine unshallow and not unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 4bf556fb3..bc3badce4 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,7 +68,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +92,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch --unshallow origin "$BASE_BRANCH" + execute_command git fetch --unshallow origin "$BASE_BRANCH" || execute_command git fetch origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +101,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" || execute_command git fetch origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From c7b84a2e7a83c3f2e349a4ada3ce1e62a70c025d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:33:12 +0300 Subject: [PATCH 056/220] Use function to fetch unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index bc3badce4..848669566 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -48,6 +48,10 @@ while [[ $# -gt 0 ]]; do esac done +git_fetch_unshallow() { + git fetch --unshallow "$@" 2>/dev/null || git fetch "$@" +} + if [ -z "$TAG" ]; then echo "Error: TAG is required as argument" echo "Usage: $0 [--allow-modify] " @@ -68,7 +72,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +96,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch --unshallow origin "$BASE_BRANCH" || execute_command git fetch origin "$BASE_BRANCH" + execute_command git_fetch_unshallow origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +105,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" || execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From dec5f36f673df35038a33c2e25879019f11af8ef Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:34:09 +0300 Subject: [PATCH 057/220] Set author for merge --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 848669566..693d363ef 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -142,6 +142,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "HEAD" git log --graph --decorate -5 + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" From 4e05f49113ba18c028861a8697d58004765e4421 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:38:45 +0300 Subject: [PATCH 058/220] Fix sha calculation --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index d258369aa..187769731 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -sfiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -sfL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From e86576dad368ef70b1b51e0acb5ee0552f3002de Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:58:18 +0300 Subject: [PATCH 059/220] Fix ensure branch call when building --- .github/workflows/pre-merge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 51a50a3a2..82c9414f8 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -44,10 +44,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure release branch - if: ${{ github.event_name == 'workflow_call' }} + if: ${{ inputs.release_tag }} uses: ./.github/actions/ensure-release-branch with: - release_tag: ${{ github.event.inputs.release_tag }} + release_tag: ${{ inputs.release_tag }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} From 017408251c8c767cc30778c1074b749b983842cf Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:58:18 +0300 Subject: [PATCH 060/220] Fix ensure branch call when building --- .github/actions/apply-docker-version/action.yml | 4 +++- .../apply-docker-version.sh | 2 -- .../actions/configure_git_commiter/action.yml | 8 ++++++++ .../actions/ensure-release-branch/action.yml | 2 ++ .../ensure-release-branch.sh | 17 ----------------- .github/workflows/release_build_and_test.yml | 8 +++++++- 6 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 .github/actions/configure_git_commiter/action.yml diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index fd02d56ba..b1fe46582 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -6,7 +6,9 @@ inputs: runs: using: "composite" steps: - - name: Ensure Release Branch + - name: Configure Git Commiter + uses: ./.github/actions/configure_git_commiter + - name: Apply docker version shell: bash run: | ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6c05391d9..f5a86d986 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -93,8 +93,6 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git diff --cached execute_command git commit -m "$TAG" diff --git a/.github/actions/configure_git_commiter/action.yml b/.github/actions/configure_git_commiter/action.yml new file mode 100644 index 000000000..3ab5c0fe0 --- /dev/null +++ b/.github/actions/configure_git_commiter/action.yml @@ -0,0 +1,8 @@ +runs: + using: "composite" + steps: + - name: Configure bot commiter + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 8426db2a8..160960ec3 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -9,6 +9,8 @@ inputs: runs: using: "composite" steps: + - name: Conigure Git Commiter + uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch shell: bash run: | diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 693d363ef..b6c7f4efa 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -127,23 +127,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." # Try to merge the release branch into the current release version branch - echo origin/$RELEASE_BRANCH - git log --graph --decorate -5 origin/$RELEASE_BRANCH - - echo origin/$RELEASE_VERSION_BRANCH - git log --graph --decorate -5 origin/$RELEASE_VERSION_BRANCH - - echo $RELEASE_BRANCH - git log --graph --decorate -5 $RELEASE_BRANCH - - echo $RELEASE_VERSION_BRANCH - git log --graph --decorate -5 $RELEASE_VERSION_BRANCH - - echo "HEAD" - git log --graph --decorate -5 - - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 10ea16c37..012342dab 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,3 +1,9 @@ +# This worjflow is a part of release automation process. +# It is intended to be run with workflow_dispatch event by the automation. + +# Warning: Workflow does switch branches and this may lead to confusion when changing workflow actions. +# The usual safety rule is to make changes to workflow or actions in base branch (e.g, release/8.X) +# Version branches (e.g, 8.0.10-rc5-int8) will merge changes from base branch automatically. on: workflow_dispatch: inputs: @@ -33,5 +39,5 @@ jobs: uses: ./.github/workflows/pre-merge.yml secrets: inherit with: - release_tag: ${{ github.event.inputs.release_tag }} + release_tag: ${{ inputs.release_tag }} From 8ea64de61dff8725b6a5f4be1f57d86b0cf43080 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 2 Aug 2025 12:56:36 +0300 Subject: [PATCH 061/220] Try to create verified commit --- .../actions/apply-docker-version/action.yml | 15 +++++++-- .../apply-docker-version.sh | 33 +++++++++++-------- .github/workflows/release_build_and_test.yml | 1 + 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index b1fe46582..3d93f8326 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -3,12 +3,23 @@ inputs: description: 'Release tag to build' required: true +outputs: + changed_files: + description: 'List of files that were modified' + value: ${{ steps.apply-version.outputs.changed_files }} + runs: using: "composite" steps: - - name: Configure Git Commiter - uses: ./.github/actions/configure_git_commiter - name: Apply docker version + id: apply-version shell: bash run: | ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} + + - name: Create verified commit + if: steps.apply-version.outputs.changed_files != '' + uses: iarekylew00t/verified-bot-commit@v1 + with: + message: ${{ inputs.release_tag }} + files: ${{ steps.apply-version.outputs.changed_files }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index f5a86d986..130c29e2d 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -77,29 +77,36 @@ update_dockerfile() { fi } -# Track if any files were modified -files_modified=false +# Track which files were modified +changed_files=() # Update debian/Dockerfile if update_dockerfile "debian/Dockerfile"; then - files_modified=true + changed_files+=("debian/Dockerfile") fi # Update alpine/Dockerfile if update_dockerfile "alpine/Dockerfile"; then - files_modified=true + changed_files+=("alpine/Dockerfile") fi -# Commit changes if any files were modified -if [ "$files_modified" = true ]; then - echo "Files were modified, committing changes..." - git add debian/Dockerfile alpine/Dockerfile - git diff --cached - execute_command git commit -m "$TAG" - execute_command git push origin "$TAG" - echo "Changes committed with message: $TAG" +# Output the list of changed files for GitHub Actions +if [ ${#changed_files[@]} -gt 0 ]; then + echo "Files were modified:" + printf '%s\n' "${changed_files[@]}" + + # Set GitHub Actions output + changed_files_output=$(printf '%s\n' "${changed_files[@]}") + { + echo "changed_files<> "$GITHUB_OUTPUT" + + echo "Changed files output set for next step" else - echo "No files were modified, nothing to commit" + echo "No files were modified" + echo "changed_files=" >> "$GITHUB_OUTPUT" fi echo "Docker version update completed" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 012342dab..3b1f96fb2 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -36,6 +36,7 @@ jobs: build-and-test: needs: prepare-release + if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From 4bb6e39033489187f67005370d0f7a6342689c6b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 2 Aug 2025 13:04:23 +0300 Subject: [PATCH 062/220] Push to release_version_branch instead of release_branch --- .github/actions/apply-docker-version/action.yml | 4 ++++ .github/actions/ensure-release-branch/action.yml | 6 ++++++ .../actions/ensure-release-branch/ensure-release-branch.sh | 3 +++ .github/workflows/release_build_and_test.yml | 2 ++ 4 files changed, 15 insertions(+) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 3d93f8326..315c56513 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -2,6 +2,9 @@ inputs: release_tag: description: 'Release tag to build' required: true + release_version_branch: + description: 'Release version branch to commit to' + required: true outputs: changed_files: @@ -23,3 +26,4 @@ runs: with: message: ${{ inputs.release_tag }} files: ${{ steps.apply-version.outputs.changed_files }} + ref: ${{ inputs.release_version_branch }} diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 160960ec3..e7a51bca0 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -6,12 +6,18 @@ inputs: description: 'Allow modifying the repository' default: false +outputs: + release_version_branch: + description: 'The release version branch name' + value: ${{ steps.ensure-branch.outputs.release_version_branch }} + runs: using: "composite" steps: - name: Conigure Git Commiter uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch + id: ensure-branch shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index b6c7f4efa..be9ccd48e 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -61,6 +61,9 @@ fi # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" +# Output the release version branch for GitHub Actions +echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" + echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3b1f96fb2..816f0b7e6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,7 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch + id: ensure-branch uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} @@ -33,6 +34,7 @@ jobs: uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} + release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: needs: prepare-release From 0fda96a2e9c206ce0e56b432b750c9d5d6b24eac Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:15:36 +0300 Subject: [PATCH 063/220] Optional workflow_uuid --- .github/workflows/release_build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 816f0b7e6..be90a71dd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -10,9 +10,13 @@ on: release_tag: description: 'Release tag to build' required: true + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false jobs: prepare-release: + name: ${{ github.event.inputs.workflow_uuid && format('prepare-release {0}', github.event.inputs.workflow_uuid) || 'prepare-release' }} runs-on: ["ubuntu-latest"] steps: - name: Checkout code From ab10d32e7c136e6b13d6d2d96cb3c571b522c40f Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:35:08 +0300 Subject: [PATCH 064/220] Use run-name --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index be90a71dd..972fc9abd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,9 +14,10 @@ on: description: 'Optional UUID to identify this workflow run' required: false +run-name: "Release Build and Test ${{ github.event.inputs.workflow_uuid && format(' {0}', github.event.inputs.workflow_uuid) || '' }}" + jobs: prepare-release: - name: ${{ github.event.inputs.workflow_uuid && format('prepare-release {0}', github.event.inputs.workflow_uuid) || 'prepare-release' }} runs-on: ["ubuntu-latest"] steps: - name: Checkout code From 8f8e26b818f0df0dba5cf787f0945d05c4e1fd1c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:38:19 +0300 Subject: [PATCH 065/220] Fix double space --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 972fc9abd..80bc78340 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,7 +14,7 @@ on: description: 'Optional UUID to identify this workflow run' required: false -run-name: "Release Build and Test ${{ github.event.inputs.workflow_uuid && format(' {0}', github.event.inputs.workflow_uuid) || '' }}" +run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" jobs: prepare-release: From 87e8b8237b4551b3428df7e3d31bc19521e99b8c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 17:33:10 +0300 Subject: [PATCH 066/220] Use ghcr.io to push images --- .github/workflows/pre-merge.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 82c9414f8..6dfdb2fb0 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -22,15 +22,15 @@ jobs: - alpine platform: - linux/amd64 - - linux/i386 - - linux/arm/v5 - - linux/arm/v6 - - linux/arm/v7 - - linux/mips64le - - linux/ppc64le - - linux/s390x - - linux/arm64 - - linux/riscv64 + # - linux/i386 + # - linux/arm/v5 + # - linux/arm/v6 + # - linux/arm/v7 + # - linux/mips64le + # - linux/ppc64le + # - linux/s390x + # - linux/arm64 + # - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -52,7 +52,7 @@ jobs: with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} - registry_username: ${{ vars.REGISTRY_USERNAME }} - registry_password: ${{ secrets.REGISTRY_PASSWORD }} + registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} + registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY }} \ No newline at end of file From c3dcc3e4aeeb9b1d23cc77d88c6acc0e9a580126 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 17:35:36 +0300 Subject: [PATCH 067/220] Enable build again for testing --- .github/workflows/release_build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 80bc78340..ea9dbb2f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,6 @@ jobs: build-and-test: needs: prepare-release - if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From d5c2133eebad8a20f441a65f806f824042d35ed3 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 19:00:45 +0300 Subject: [PATCH 068/220] Try to fix ghcr.io tag --- .github/actions/build-and-tag-locally/action.yml | 15 +++++++++------ .github/workflows/pre-merge.yml | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 35f234b2f..78175ce86 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -19,6 +19,9 @@ inputs: registry_repository: description: 'Repository to push the image to' required: false + release_tag: + description: 'Release tag to build' + required: false runs: using: "composite" @@ -102,7 +105,7 @@ runs: shell: bash run: | docker save -o /tmp/image-${{ steps.platform.outputs.display_name }}.tar ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Upload image uses: actions/upload-artifact@v4 with: @@ -115,7 +118,7 @@ runs: if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} run: | docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Container Logs if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -128,7 +131,7 @@ runs: run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server - + - name: Verify installed modules if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -148,7 +151,7 @@ runs: echo "The following modules are missing: ${missing_modules[*]}" exit 1 fi - + - name: Test RedisBloom if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -158,7 +161,7 @@ runs: [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:hash")" = "1" ] || { echo "RedisBloom test failed: 'redis:hash' not found"; exit 1; } [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:list")" = "0" ] || { echo "RedisBloom test failed: 'redis:list' found unexpectedly"; exit 1; } echo "RedisBloom test passed successfully" - + - name: Test RediSearch if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -230,6 +233,6 @@ runs: with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '' }}${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6dfdb2fb0..f3f27f5d7 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,4 +55,4 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY }} \ No newline at end of file + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} From 81b938aafe667029ed060d5bbd03a6eb559accdb Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 20:49:37 +0300 Subject: [PATCH 069/220] Fix release_tag propagation --- .github/workflows/pre-merge.yml | 2 ++ .github/workflows/release_build_and_test.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f3f27f5d7..1f82c5538 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -56,3 +56,5 @@ jobs: registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + release_tag: ${{ inputs.release_tag }} + diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ea9dbb2f6..e6d2405d6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -46,5 +46,5 @@ jobs: uses: ./.github/workflows/pre-merge.yml secrets: inherit with: - release_tag: ${{ inputs.release_tag }} + release_tag: ${{ github.event.inputs.release_tag }} From e21a84d3621957093d90b1384402d40ecf79659b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 20:51:28 +0300 Subject: [PATCH 070/220] Fix unexpected end of expression --- .github/actions/build-and-tag-locally/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 78175ce86..79db0e772 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -233,6 +233,6 @@ runs: with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '' }}${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha cache-to: type=gha,mode=max From f26e8eac9f613b3865da3847a9f168772909ba78 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 21:30:46 +0300 Subject: [PATCH 071/220] registry tags must be lowercase --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 1f82c5538..55e01b788 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,6 +55,6 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', toLower(github.repository)) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} From 7db9d6693996ee0c5708976fdb4a685d26c4238d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:28:51 +0300 Subject: [PATCH 072/220] Step to correctly format registry tag --- .github/actions/build-and-tag-locally/action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 79db0e772..472a17a07 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -227,12 +227,22 @@ runs: path: test/report-entrypoint.xml reporter: java-junit + - name: Format registry tag + id: format-registry-tag + run: | + printf "tag=%s:%s%s-%s" \ + "${{ inputs.registry_repository }}" \ + "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \ + "${{ github.sha }}" \ + "${{ inputs.distribution }}" \ + | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" + - name: Push image uses: docker/build-push-action@v6 if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ steps.format-registry-tag.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max From 507d27ae916293905832b8c02b575d10b374e967 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:30:37 +0300 Subject: [PATCH 073/220] Remove unsupported toLower() --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 55e01b788..1f82c5538 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,6 +55,6 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', toLower(github.repository)) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} From 5276cb94c0e3022fff2796d0fe9aa806ae2429ab Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:32:07 +0300 Subject: [PATCH 074/220] Fix step shell --- .github/actions/build-and-tag-locally/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 472a17a07..060384d5b 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -229,6 +229,7 @@ runs: - name: Format registry tag id: format-registry-tag + shell: bash run: | printf "tag=%s:%s%s-%s" \ "${{ inputs.registry_repository }}" \ From be7d64252403cdd534d5bf3e623739996150a38d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:14:33 +0300 Subject: [PATCH 075/220] Try verified merge --- .../ensure-release-branch.sh | 37 ++++++++++++++++--- .github/workflows/release_build_and_test.yml | 1 + 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index be9ccd48e..547ae9fe4 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -127,12 +127,37 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Changes detected but refusing to merge without --allow-modify option" exit 1 fi - - echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." - # Try to merge the release branch into the current release version branch - execute_command git merge "origin/$RELEASE_BRANCH" --no-edit - # Push the merged changes to origin - execute_command git push origin "$RELEASE_VERSION_BRANCH" + # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) + API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" + + read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:16:18 +0300 Subject: [PATCH 076/220] Fix heredoc --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 547ae9fe4..4572b4d14 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -132,11 +132,11 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:19:53 +0300 Subject: [PATCH 077/220] Show commits --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 4572b4d14..fb77db9f5 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -122,6 +122,9 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then if [ "$COMMITS_BEHIND" -gt 0 ]; then echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + console_output 1 gray "Commits mising:" + console_output 1 gray "$last_cmd_stdout" if [ -z "$ALLOW_MODIFY" ]; then echo "Changes detected but refusing to merge without --allow-modify option" From baf14b24b12b4abd0cc2ab6cf2e09d5edc6cea5b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:25:50 +0300 Subject: [PATCH 078/220] Debug --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index fb77db9f5..76e99722d 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,6 +121,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then + set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" From af2fa706a743df2ee0b8b2d1ef818d8c6658b96c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:27:37 +0300 Subject: [PATCH 079/220] debg --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 76e99722d..a000fed05 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,6 +121,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then + # debug set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From bcb994bb8d737c7a33c2f29800c27cc9d91e7144 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:28:18 +0300 Subject: [PATCH 080/220] Format oneline --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index a000fed05..870b53f8b 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -124,7 +124,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # debug set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" - execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" console_output 1 gray "$last_cmd_stdout" From 52f21d98052065ecb3c136c179ffb88b36e9c645 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:35:58 +0300 Subject: [PATCH 081/220] Remove heredoc --- .../ensure-release-branch/ensure-release-branch.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 870b53f8b..d9510bc3f 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,13 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" - read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:42:06 +0300 Subject: [PATCH 082/220] debug3 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index d9510bc3f..a8e8e710a 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,6 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" + # t PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" # Make the request and capture status code + body From 0dd9300cd7225a97a662c5dfb856c5cf86dd7b40 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:44:57 +0300 Subject: [PATCH 083/220] debug 4 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index a8e8e710a..97b96695c 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,8 +135,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" - # t PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" + echo "$PAYLOAD" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ From debc6a172b099f5ee1d56025bb09eb93b7e26098 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:47:35 +0300 Subject: [PATCH 084/220] debyg 5 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 97b96695c..755d8ed59 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,6 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" + echo gh_token=$(gh auth token) PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" echo "$PAYLOAD" From 3d5e032728907b173e30f425c551eacd5a7bb409 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:52:08 +0300 Subject: [PATCH 085/220] Add GITHUB_TOKEN --- .github/actions/ensure-release-branch/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index e7a51bca0..38f0edf03 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -18,6 +18,8 @@ runs: uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch id: ensure-branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} From a9e5e2598934fa501a0fb6080e7ad627d4ae87df Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:56:19 +0300 Subject: [PATCH 086/220] Pass gh_token --- .github/actions/ensure-release-branch/action.yml | 5 ++++- .github/workflows/pre-merge.yml | 1 + .github/workflows/release_build_and_test.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 38f0edf03..8daebfb97 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -5,6 +5,9 @@ inputs: allow_modify: description: 'Allow modifying the repository' default: false + gh_token: + description: 'GitHub token to use' + required: true outputs: release_version_branch: @@ -19,7 +22,7 @@ runs: - name: Ensure Release Branch id: ensure-branch env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 1f82c5538..8494c316b 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -48,6 +48,7 @@ jobs: uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ inputs.release_tag }} + gh_token: ${{ secrets.GITHUB_TOKEN }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 1992c2a63..9c39623a1 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -34,6 +34,7 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true + gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Apply Docker Version uses: ./.github/actions/apply-docker-version From e708852a6a60c1c5fca72c8430673124ef9b0a7f Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:02:22 +0300 Subject: [PATCH 087/220] debug 6 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 755d8ed59..d9510bc3f 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,9 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" - echo gh_token=$(gh auth token) PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" - echo "$PAYLOAD" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ From c266e09acac58c72b66ec750963c518bd52cce5b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:08:16 +0300 Subject: [PATCH 088/220] fix tkn --- .github/actions/ensure-release-branch/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 8daebfb97..814ba6059 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -22,7 +22,7 @@ runs: - name: Ensure Release Branch id: ensure-branch env: - GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.gh_token }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} From 4ddc546d00397fd93bcf3f57132bd7b5168f6f5d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:09:07 +0300 Subject: [PATCH 089/220] debug 7 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index d9510bc3f..7943f7d20 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -132,6 +132,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Changes detected but refusing to merge without --allow-modify option" exit 1 fi + # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" From 1b671ac9a90e6c6cf5d9b203448d39a3897c0c72 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:13:40 +0300 Subject: [PATCH 090/220] remove set -x --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 7943f7d20..e4b940838 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,8 +121,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then - # debug - set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" From 8658fa6a22a5399b6dc8c9bb10170bcca3981152 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:10:03 +0300 Subject: [PATCH 091/220] Moved verfied merge into function --- .github/actions/common/github_helpers.sh | 46 +++++++++++++++++++ .../ensure-release-branch.sh | 39 ++++------------ 2 files changed, 54 insertions(+), 31 deletions(-) create mode 100644 .github/actions/common/github_helpers.sh diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh new file mode 100644 index 000000000..5f67fe089 --- /dev/null +++ b/.github/actions/common/github_helpers.sh @@ -0,0 +1,46 @@ +github_create_verified_merge() { + BASE_BRANCH= + HEAD_BRANCH= + while [[ $# -gt 0 ]]; do + case $1 in + --from) + HEAD_BRANCH="$2" + shift + ;; + --to) + BASE_BRANCH="$2" + shift + ;; + -*) + echo "Error: Unknown option $1" + exit 1 + ;; + esac + done + + if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then + echo "Error: Missing required arguments --from and --to" + exit 1 + fi + + # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) + API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" + + PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" + + # Make the request and capture status code + body + HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$API_URL" \ + -d "$PAYLOAD") + + case "$HTTP_CODE" in + 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; + 204) echo "āœ”ļø Already up to date (no merge necessary)";; + 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; + *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; + esac +} \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index e4b940838..0e67e0b69 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -21,6 +21,8 @@ VERBOSITY=1 SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/github_helpers.sh" # Parse arguments ALLOW_MODIFY="" @@ -109,8 +111,6 @@ fi execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "$RELEASE_VERSION_BRANCH" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." @@ -119,11 +119,9 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') - if [ "$COMMITS_BEHIND" -gt 0 ]; then - echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH:" execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - console_output 1 gray "Commits mising:" console_output 1 gray "$last_cmd_stdout" if [ -z "$ALLOW_MODIFY" ]; then @@ -131,35 +129,14 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then exit 1 fi - # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) - API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" - - PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" - - # Make the request and capture status code + body - HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "$API_URL" \ - -d "$PAYLOAD") - - case "$HTTP_CODE" in - 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; - 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; - esac - - # Update the runner's working copy to the server-side result (optional but handy) - git fetch origin - git checkout "${RELEASE_VERSION_BRANCH}" - git reset --hard "origin/${RELEASE_VERSION_BRANCH}" + github_create_verified_merge --from "$RELEASE_BRANCH" --to "$RELEASE_VERSION_BRANCH" fi - exit 0 + execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "${RELEASE_VERSION_BRANCH}" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + exit 0 fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" From 89d5e41fbbcb159a0ae5b6233eff574edb54cc77 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:52:44 +0300 Subject: [PATCH 092/220] Just a change --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 0e67e0b69..6cfd998b1 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -115,7 +115,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" - # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') From 3a29c1b9daa36ba0120777b84b519517a9391162 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:58:03 +0300 Subject: [PATCH 093/220] Fix arguments parsing --- .github/actions/common/github_helpers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index 5f67fe089..dc4450d5e 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -11,8 +11,8 @@ github_create_verified_merge() { BASE_BRANCH="$2" shift ;; - -*) - echo "Error: Unknown option $1" + *) + echo "Error: Unknown argument $1" exit 1 ;; esac From f9bedc14095ca64048a5f2f015ee0ad8ad283391 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:02:02 +0300 Subject: [PATCH 094/220] Add second shift --- .github/actions/common/github_helpers.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index dc4450d5e..ef193603d 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -6,10 +6,12 @@ github_create_verified_merge() { --from) HEAD_BRANCH="$2" shift + shift ;; --to) BASE_BRANCH="$2" shift + shift ;; *) echo "Error: Unknown argument $1" From 71a636219c04d0382aca36974465cf24e8db3360 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:39:13 +0300 Subject: [PATCH 095/220] Fix docker files updating --- .../apply-docker-version.sh | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 130c29e2d..5ca21d469 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -57,38 +57,30 @@ update_dockerfile() { # Update REDIS_DOWNLOAD_URL if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile" - updated=true - echo " Updated REDIS_DOWNLOAD_URL" + else + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_URL not found" + return 1 fi + # Update REDIS_DOWNLOAD_SHA if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile" - updated=true - echo " Updated REDIS_DOWNLOAD_SHA" - fi - - if [ "$updated" = true ]; then - echo " $dockerfile updated successfully" - return 0 else - echo " No changes needed in $dockerfile" + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_SHA not found" return 1 fi } +docker_files=("debian/Dockerfile" "alpine/Dockerfile") # Track which files were modified changed_files=() -# Update debian/Dockerfile -if update_dockerfile "debian/Dockerfile"; then - changed_files+=("debian/Dockerfile") -fi +for dockerfile in "${docker_files[@]}"; do + update_dockerfile "$dockerfile" +done -# Update alpine/Dockerfile -if update_dockerfile "alpine/Dockerfile"; then - changed_files+=("alpine/Dockerfile") -fi +changed_files=($(git diff --name-only "${docker_files[@]}")) # Output the list of changed files for GitHub Actions if [ ${#changed_files[@]} -gt 0 ]; then @@ -107,6 +99,4 @@ if [ ${#changed_files[@]} -gt 0 ]; then else echo "No files were modified" echo "changed_files=" >> "$GITHUB_OUTPUT" -fi - -echo "Docker version update completed" +fi \ No newline at end of file From 4e317e62e5fc48d7b97632b48e97908e360ee5c2 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:44:36 +0300 Subject: [PATCH 096/220] Test modification --- debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/Dockerfile b/debian/Dockerfile index 0e7d71718..ad615c384 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 5aa2d5dcc9b2a3374059d8037eeabf60ac0d4c51 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 12:13:33 +0300 Subject: [PATCH 097/220] Merge back --- .../actions/ensure-release-branch/action.yml | 5 +- .../ensure-release-branch.sh | 2 +- .../merge-branches-verified/action.yml | 15 ++++++ .../merge-branches-verified.sh | 49 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 26 +++++++++- 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 .github/actions/merge-branches-verified/action.yml create mode 100644 .github/actions/merge-branches-verified/merge-branches-verified.sh diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 814ba6059..50e7db536 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -11,8 +11,11 @@ inputs: outputs: release_version_branch: - description: 'The release version branch name' + description: 'The release version branch name, e.g.: 8.X.X-int1' value: ${{ steps.ensure-branch.outputs.release_version_branch }} + release_branch: + description: 'The release branch name, e.g.: release/8.X' + value: ${{ steps.ensure-branch.outputs.release_branch }} runs: using: "composite" diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 6cfd998b1..ac51293a0 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -63,7 +63,6 @@ fi # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" -# Output the release version branch for GitHub Actions echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" echo "TAG: $TAG" @@ -72,6 +71,7 @@ echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" +echo "release_branch=$RELEASE_BRANCH" >> "$GITHUB_OUTPUT" # Check if RELEASE_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_BRANCH" diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml new file mode 100644 index 000000000..56465948f --- /dev/null +++ b/.github/actions/merge-branches-verified/action.yml @@ -0,0 +1,15 @@ +inputs: + from_branch: + description: 'Branch to merge from' + required: true + to_branch: + description: 'Branch to merge into' + required: true + +runs: + using: "composite" + steps: + - name: Do verified merge + shell: bash + run: | + ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh new file mode 100644 index 000000000..025fd142c --- /dev/null +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e + +# This script updates Redis version in Dockerfiles using environment variables +# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. + +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/helpers.sh" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/github_helpers.sh" + +FROM_BRANCH= +TO_BRANCH= +while [[ $# -gt 0 ]]; do + case $1 in + --from) + FROM_BRANCH="$2" + shift + shift + ;; + --to) + TO_BRANCH="$2" + shift + shift + ;; + *) + echo "Error: Unknown argument $1" + exit 1 + ;; + esac +done + +if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then + echo "Error: Missing required arguments --from and --to" + exit 1 +fi + +execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 9c39623a1..6e709bd9e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,4 +1,4 @@ -# This worjflow is a part of release automation process. +# This workflow is a part of release automation process. # It is intended to be run with workflow_dispatch event by the automation. # Warning: Workflow does switch branches and this may lead to confusion when changing workflow actions. @@ -19,6 +19,8 @@ run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format jobs: prepare-release: runs-on: ["ubuntu-latest"] + outputs: + changed_files: ${{ steps.apply-version.outputs.changed_files }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -37,6 +39,7 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Apply Docker Version + id: apply-version uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} @@ -50,3 +53,24 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} + merge-back-to-release-branch: + needs: [prepare-release, build-and-test] + if: success() && needs.prepare-release.outputs.changed_files != '' + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure Release Branch + id: ensure-branch + uses: ./.github/actions/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: false + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge back to release branch + uses: ./.github/actions/merge-branches-verified + with: + from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} + to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From c4121f6035d0b1cbcd71ed99b257296d97d48c2a Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 12:16:43 +0300 Subject: [PATCH 098/220] Enable build --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6e709bd9e..105018a82 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -47,7 +47,7 @@ jobs: build-and-test: needs: prepare-release - if: false + #if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From d54f8acaf0fb858073dfc115ab0ed9075c91f1b0 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:04:39 +0300 Subject: [PATCH 099/220] Set execute bit --- .../actions/merge-branches-verified/merge-branches-verified.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/merge-branches-verified/merge-branches-verified.sh diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh old mode 100644 new mode 100755 From ee242c2c778449fc68e7921a28f27399ae2936ce Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:23:51 +0300 Subject: [PATCH 100/220] Add debug --- .github/actions/common/github_helpers.sh | 1 + .../merge-branches-verified/merge-branches-verified.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index ef193603d..ae225524a 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -1,4 +1,5 @@ github_create_verified_merge() { + set -x BASE_BRANCH= HEAD_BRANCH= while [[ $# -gt 0 ]]; do diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 025fd142c..7965f1d9c 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e - -# This script updates Redis version in Dockerfiles using environment variables -# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. +set -x # shellcheck disable=SC2034 last_cmd_stdout="" From 3e47704f39b22c57eda9413d2c0feffd532e59e7 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:47:27 +0300 Subject: [PATCH 101/220] Skip build for debug --- .../merge-branches-verified.sh | 2 +- .github/workflows/release_build_and_test.yml | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 7965f1d9c..05029c304 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -44,4 +44,4 @@ if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then exit 1 fi -execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file +github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 105018a82..ff52de8cd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -47,11 +47,17 @@ jobs: build-and-test: needs: prepare-release + runs-on: ["ubuntu-latest"] #if: false - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} + #uses: ./.github/workflows/pre-merge.yml + #secrets: inherit + #with: + #release_tag: ${{ github.event.inputs.release_tag }} + steps: + - name: Test + run: | + echo Poop + merge-back-to-release-branch: needs: [prepare-release, build-and-test] From bda1955500f8c3911acd1e42f4ab9a7eea717766 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:48:08 +0300 Subject: [PATCH 102/220] Disable fail on error --- .../actions/merge-branches-verified/merge-branches-verified.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 05029c304..75eb9e77b 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +#set -e set -x # shellcheck disable=SC2034 From b6d53ec2547dbc19a8daf87ada54a54504241ce4 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:55:04 +0300 Subject: [PATCH 103/220] Further debug --- .github/actions/common/github_helpers.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index ae225524a..2e315e765 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -31,6 +31,7 @@ github_create_verified_merge() { PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" + echo "Going to debug this" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ -X POST \ @@ -40,6 +41,8 @@ github_create_verified_merge() { "$API_URL" \ -d "$PAYLOAD") + echo "HTTP_CODE: $HTTP_CODE" + case "$HTTP_CODE" in 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; 204) echo "āœ”ļø Already up to date (no merge necessary)";; From 28f12813e93e91ffe0482b1a497e203be901b000 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:06:47 +0300 Subject: [PATCH 104/220] Fix merge function and add gh token --- .github/actions/common/github_helpers.sh | 12 ++++-------- .github/actions/common/helpers.sh | 4 +++- .github/actions/merge-branches-verified/action.yml | 5 +++++ .../merge-branches-verified.sh | 6 ++---- .github/workflows/release_build_and_test.yml | 1 + 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index 2e315e765..fe6d09adf 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -1,5 +1,4 @@ github_create_verified_merge() { - set -x BASE_BRANCH= HEAD_BRANCH= while [[ $# -gt 0 ]]; do @@ -16,14 +15,14 @@ github_create_verified_merge() { ;; *) echo "Error: Unknown argument $1" - exit 1 + return 1 ;; esac done if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then echo "Error: Missing required arguments --from and --to" - exit 1 + return 1 fi # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) @@ -31,7 +30,6 @@ github_create_verified_merge() { PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" - echo "Going to debug this" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ -X POST \ @@ -41,12 +39,10 @@ github_create_verified_merge() { "$API_URL" \ -d "$PAYLOAD") - echo "HTTP_CODE: $HTTP_CODE" - case "$HTTP_CODE" in 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; + 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; return 1;; + *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; return 1;; esac } \ No newline at end of file diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh index 3a74f3062..cf0039fcd 100644 --- a/.github/actions/common/helpers.sh +++ b/.github/actions/common/helpers.sh @@ -1,6 +1,6 @@ # Function to execute command from array and capture output execute_command() { -# turn off errexit (set -e) if it is active and restore it later + # turn off errexit (set -e) if it is active and restore it later echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" local cmd @@ -26,6 +26,7 @@ execute_command() { # Execute command and capture output console_output 1 gray "Executing command: ${cmd[*]}" + "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" last_cmd_result=$? @@ -33,6 +34,7 @@ execute_command() { last_cmd_stdout=$(cat "$stdout_file") last_cmd_stderr=$(cat "$stderr_file") + if [ "$last_cmd_result" -ne 0 ]; then console_output 0 red "Command failed with exit code $last_cmd_result" console_output 0 red "Standard Output:" diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml index 56465948f..a5c1c95f4 100644 --- a/.github/actions/merge-branches-verified/action.yml +++ b/.github/actions/merge-branches-verified/action.yml @@ -1,4 +1,7 @@ inputs: + gh_token: + description: 'GitHub token to use' + required: true from_branch: description: 'Branch to merge from' required: true @@ -8,6 +11,8 @@ inputs: runs: using: "composite" + env: + GITHUB_TOKEN: ${{ inputs.gh_token }} steps: - name: Do verified merge shell: bash diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 75eb9e77b..597a9ccb0 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,6 +1,5 @@ #!/bin/bash -#set -e -set -x +set -e # shellcheck disable=SC2034 last_cmd_stdout="" @@ -11,7 +10,6 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 - SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" @@ -44,4 +42,4 @@ if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then exit 1 fi -github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file +execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ff52de8cd..caee9e83e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -80,3 +80,4 @@ jobs: with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} + gh_token: ${{ secrets.GITHUB_TOKEN }} From f6bbb64bb973978d10dc884006aa57fa3e14a3a2 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:09:37 +0300 Subject: [PATCH 105/220] Fix env --- .github/actions/merge-branches-verified/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml index a5c1c95f4..669312a76 100644 --- a/.github/actions/merge-branches-verified/action.yml +++ b/.github/actions/merge-branches-verified/action.yml @@ -11,10 +11,10 @@ inputs: runs: using: "composite" - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} steps: - name: Do verified merge + env: + GITHUB_TOKEN: ${{ inputs.gh_token }} shell: bash run: | ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file From 970c2aa9435fdd1b4d47ba0dab49bb4a25c6edfb Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:42:45 +0300 Subject: [PATCH 106/220] Move reusable actions to redis-oss-release-automation --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 8494c316b..e35821d27 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -45,11 +45,11 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/build-and-tag-locally + - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index caee9e83e..2321628c3 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -26,13 +26,13 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive + uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -69,14 +69,14 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch - uses: ./.github/actions/merge-branches-verified + uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From 707d66badf9f4cf3a86f968683dc793f6a2b8873 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:49:40 +0300 Subject: [PATCH 107/220] Fix actions refs --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index e35821d27..164357a44 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -45,11 +45,11 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally + - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally@main with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 2321628c3..a87e3bd1e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -26,13 +26,13 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive + uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -69,14 +69,14 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch - uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified + uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From 5d1810f2d0d502fe1d921c8e92d13940c23671d6 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:26:08 +0300 Subject: [PATCH 108/220] Remove actions that were transerred to release automation --- .github/actions/common/github_helpers.sh | 48 ------ .github/actions/common/helpers.sh | 85 ---------- .../actions/configure_git_commiter/action.yml | 8 - .../actions/ensure-release-branch/action.yml | 31 ---- .../ensure-release-branch.sh | 156 ------------------ .../merge-branches-verified/action.yml | 20 --- .../merge-branches-verified.sh | 45 ----- .../validate-redis-release-archive/action.yml | 12 -- .../validate-redis-release-archive.sh | 47 ------ 9 files changed, 452 deletions(-) delete mode 100644 .github/actions/common/github_helpers.sh delete mode 100644 .github/actions/common/helpers.sh delete mode 100644 .github/actions/configure_git_commiter/action.yml delete mode 100644 .github/actions/ensure-release-branch/action.yml delete mode 100755 .github/actions/ensure-release-branch/ensure-release-branch.sh delete mode 100644 .github/actions/merge-branches-verified/action.yml delete mode 100755 .github/actions/merge-branches-verified/merge-branches-verified.sh delete mode 100644 .github/actions/validate-redis-release-archive/action.yml delete mode 100755 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh deleted file mode 100644 index fe6d09adf..000000000 --- a/.github/actions/common/github_helpers.sh +++ /dev/null @@ -1,48 +0,0 @@ -github_create_verified_merge() { - BASE_BRANCH= - HEAD_BRANCH= - while [[ $# -gt 0 ]]; do - case $1 in - --from) - HEAD_BRANCH="$2" - shift - shift - ;; - --to) - BASE_BRANCH="$2" - shift - shift - ;; - *) - echo "Error: Unknown argument $1" - return 1 - ;; - esac - done - - if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then - echo "Error: Missing required arguments --from and --to" - return 1 - fi - - # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) - API_URL="https://api.github.com/repos/$%7BGITHUB_REPOSITORY%7D/merges" - - PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" - - # Make the request and capture status code + body - HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "$API_URL" \ - -d "$PAYLOAD") - - case "$HTTP_CODE" in - 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; - 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; return 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; return 1;; - esac -} \ No newline at end of file diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh deleted file mode 100644 index cf0039fcd..000000000 --- a/.github/actions/common/helpers.sh +++ /dev/null @@ -1,85 +0,0 @@ -# Function to execute command from array and capture output -execute_command() { - # turn off errexit (set -e) if it is active and restore it later - echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" - - local cmd - - # Check if no arguments provided - if [ $# -eq 0 ]; then - # Check if cmd_array variable exists and is an array - if declare -p cmd_array 2>/dev/null | grep -q "declare -a"; then - # Use the existing cmd_array variable - cmd=("${cmd_array[@]}") - else - echo "Error: No arguments provided and cmd_array variable not found or not an array" >&2 - return 1 - fi - else - cmd=("$@") - fi - - # Create temporary files for stdout and stderr - local stdout_file stderr_file - stdout_file=$(mktemp) - stderr_file=$(mktemp) - - # Execute command and capture output - console_output 1 gray "Executing command: ${cmd[*]}" - - "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" - last_cmd_result=$? - - # Read captured output - last_cmd_stdout=$(cat "$stdout_file") - last_cmd_stderr=$(cat "$stderr_file") - - - if [ "$last_cmd_result" -ne 0 ]; then - console_output 0 red "Command failed with exit code $last_cmd_result" - console_output 0 red "Standard Output:" - console_output 0 red "$last_cmd_stdout" - console_output 0 red "Standard Error:" - console_output 0 red "$last_cmd_stderr" - fi - - # Clean up temporary files - rm -f "$stdout_file" "$stderr_file" - - [ "$restore_errexit" ] && set -e - return $last_cmd_result -} - -# Helper function to output multiline variables with color -console_output() { - local verbosity_level="$1" - local color="$2" - local content="$3" - local current_verbosity="${VERBOSITY:-0}" - - # Check if we should output based on verbosity level - if [ "$current_verbosity" -ge "$verbosity_level" ]; then - local color_code="" - local reset_code="\033[0m" - - case "$color" in - "gray"|"grey") - color_code="\033[90m" - ;; - "white") - color_code="\033[97m" - ;; - "red") - color_code="\033[91m" - ;; - *) - color_code="\033[0m" # Default to no color - ;; - esac - - # Output each line with 4-space indent and color - while IFS= read -r line || [ -n "$line" ]; do - printf "${color_code} %s${reset_code}\n" "$line" - done <<< "$content" - fi -} diff --git a/.github/actions/configure_git_commiter/action.yml b/.github/actions/configure_git_commiter/action.yml deleted file mode 100644 index 3ab5c0fe0..000000000 --- a/.github/actions/configure_git_commiter/action.yml +++ /dev/null @@ -1,8 +0,0 @@ -runs: - using: "composite" - steps: - - name: Configure bot commiter - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml deleted file mode 100644 index 50e7db536..000000000 --- a/.github/actions/ensure-release-branch/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -inputs: - release_tag: - description: 'Release tag to build' - required: true - allow_modify: - description: 'Allow modifying the repository' - default: false - gh_token: - description: 'GitHub token to use' - required: true - -outputs: - release_version_branch: - description: 'The release version branch name, e.g.: 8.X.X-int1' - value: ${{ steps.ensure-branch.outputs.release_version_branch }} - release_branch: - description: 'The release branch name, e.g.: release/8.X' - value: ${{ steps.ensure-branch.outputs.release_branch }} - -runs: - using: "composite" - steps: - - name: Conigure Git Commiter - uses: ./.github/actions/configure_git_commiter - - name: Ensure Release Branch - id: ensure-branch - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} - shell: bash - run: | - ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh deleted file mode 100755 index ac51293a0..000000000 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -# This script ensures that a release branch and release version branch exist for a given release tag. -# It allow-modifys and pushes both branches if they do not exist. -# It also checks out the release version branch at the end. -# https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation - -set -e -#set -x - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -VERBOSITY=1 - - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/github_helpers.sh" - -# Parse arguments -ALLOW_MODIFY="" -TAG="" - -while [[ $# -gt 0 ]]; do - case $1 in - --allow-modify) - ALLOW_MODIFY=1 - shift - ;; - -*) - echo "Error: Unknown option $1" - exit 1 - ;; - *) - if [ -z "$TAG" ]; then - TAG="$1" - else - echo "Error: Multiple TAG arguments provided" - exit 1 - fi - shift - ;; - esac -done - -git_fetch_unshallow() { - git fetch --unshallow "$@" 2>/dev/null || git fetch "$@" -} - -if [ -z "$TAG" ]; then - echo "Error: TAG is required as argument" - echo "Usage: $0 [--allow-modify] " - exit 1 -fi - -# Define RELEASE_VERSION_BRANCH which is the same as TAG -RELEASE_VERSION_BRANCH="$TAG" - -echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" - -echo "TAG: $TAG" -echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" - -# Detect RELEASE_BRANCH name (release/X.Y format) -RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" -echo "RELEASE_BRANCH: $RELEASE_BRANCH" -echo "release_branch=$RELEASE_BRANCH" >> "$GITHUB_OUTPUT" - -# Check if RELEASE_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then - echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" -else - echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" - if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 - fi - - # Detect base branch (previous existing branch for the version) - MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') - MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) - - # Find the previous existing release branch - execute_command git ls-remote --heads origin "release/$MAJOR.[0-9]" - BASE_BRANCH=$(echo "$last_cmd_stdout" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) - - if [ -z "$BASE_BRANCH" ]; then - echo "Error: Could not find a base branch for $RELEASE_BRANCH" - exit 1 - fi - - echo "Using base branch: $BASE_BRANCH" - - # Create new branch based on base branch and push to origin - execute_command git_fetch_unshallow origin "$BASE_BRANCH" - execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - execute_command git push origin HEAD:"$RELEASE_BRANCH" - echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" -fi - -# Check if RELEASE_VERSION_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - - # Check if there are changes in release branch that are not in release version branch - echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" - # Compare the two branches to see if there are commits in release branch not in release version branch - execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') - if [ "$COMMITS_BEHIND" -gt 0 ]; then - echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH:" - execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - console_output 1 gray "$last_cmd_stdout" - - if [ -z "$ALLOW_MODIFY" ]; then - echo "Changes detected but refusing to merge without --allow-modify option" - exit 1 - fi - - github_create_verified_merge --from "$RELEASE_BRANCH" --to "$RELEASE_VERSION_BRANCH" - fi - - execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "${RELEASE_VERSION_BRANCH}" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" - - exit 0 -fi - -echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 -fi - -execute_command git checkout "$RELEASE_BRANCH" -# At this point, we should be on RELEASE_BRANCH -echo "Current branch: $(git branch --show-current)" - -# Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin -execute_command git checkout -b "$RELEASE_VERSION_BRANCH" -execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" -echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" - -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml deleted file mode 100644 index 669312a76..000000000 --- a/.github/actions/merge-branches-verified/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -inputs: - gh_token: - description: 'GitHub token to use' - required: true - from_branch: - description: 'Branch to merge from' - required: true - to_branch: - description: 'Branch to merge into' - required: true - -runs: - using: "composite" - steps: - - name: Do verified merge - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} - shell: bash - run: | - ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh deleted file mode 100755 index 597a9ccb0..000000000 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -e - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -VERBOSITY=1 - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/github_helpers.sh" - -FROM_BRANCH= -TO_BRANCH= -while [[ $# -gt 0 ]]; do - case $1 in - --from) - FROM_BRANCH="$2" - shift - shift - ;; - --to) - TO_BRANCH="$2" - shift - shift - ;; - *) - echo "Error: Unknown argument $1" - exit 1 - ;; - esac -done - -if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then - echo "Error: Missing required arguments --from and --to" - exit 1 -fi - -execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/action.yml b/.github/actions/validate-redis-release-archive/action.yml deleted file mode 100644 index fcd775ce7..000000000 --- a/.github/actions/validate-redis-release-archive/action.yml +++ /dev/null @@ -1,12 +0,0 @@ -inputs: - release_tag: - description: 'Release tag to build' - required: true - -runs: - using: "composite" - steps: - - name: Ensure Release Branch - shell: bash - run: | - ${{ github.action_path }}/validate-redis-release-archive.sh ${{ inputs.release_tag }} \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh deleted file mode 100755 index 187769731..000000000 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -# This script validates a Redis release archive by downloading it and calculating its SHA256 sum. -# It constructs the URL, downloads the file, calculates the hash, and exports environment variables. - -# Input TAG is expected in $1 -TAG="$1" - -if [ -z "$TAG" ]; then - echo "Error: TAG is required as first argument" - exit 1 -fi - -# Construct Redis archive URL -REDIS_ARCHIVE_URL="https://github.com/redis/redis/archive/refs/tags/$%7BTAG%7D.tar.gz" -echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" - -# Download the Redis archive -TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" -echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -sfL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then - echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" - exit 1 -fi - -# Calculate SHA256 sum -echo "Calculating SHA256 sum..." -REDIS_ARCHIVE_SHA=$(sha256sum "$TEMP_ARCHIVE" | cut -d' ' -f1) -echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" - -# Write variables to GITHUB_ENV -if [ -n "$GITHUB_ENV" ]; then - echo "REDIS_ARCHIVE_URL=$REDIS_ARCHIVE_URL" >> "$GITHUB_ENV" - echo "REDIS_ARCHIVE_SHA=$REDIS_ARCHIVE_SHA" >> "$GITHUB_ENV" - echo "Environment variables written to $GITHUB_ENV" -else - echo "Error: GITHUB_ENV not set" - # Clean up temporary file - rm -f "$TEMP_ARCHIVE" - exit 1 -fi - -# Clean up temporary file -rm -f "$TEMP_ARCHIVE" - -echo "Redis archive validation completed successfully" \ No newline at end of file From 68bfae02286141a329978b24235f7addba7ef142 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:29:45 +0300 Subject: [PATCH 109/220] Return commented out actions --- .github/workflows/pre-merge.yml | 18 +++++++++--------- .github/workflows/release_build_and_test.yml | 16 +++++----------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 164357a44..bf8e71bbb 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -22,15 +22,15 @@ jobs: - alpine platform: - linux/amd64 - # - linux/i386 - # - linux/arm/v5 - # - linux/arm/v6 - # - linux/arm/v7 - # - linux/mips64le - # - linux/ppc64le - # - linux/s390x - # - linux/arm64 - # - linux/riscv64 + - linux/i386 + - linux/arm/v5 + - linux/arm/v6 + - linux/arm/v7 + - linux/mips64le + - linux/ppc64le + - linux/s390x + - linux/arm64 + - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a87e3bd1e..26e7a46bf 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,6 +14,7 @@ on: description: 'Optional UUID to identify this workflow run' required: false +# UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" jobs: @@ -47,17 +48,10 @@ jobs: build-and-test: needs: prepare-release - runs-on: ["ubuntu-latest"] - #if: false - #uses: ./.github/workflows/pre-merge.yml - #secrets: inherit - #with: - #release_tag: ${{ github.event.inputs.release_tag }} - steps: - - name: Test - run: | - echo Poop - + uses: ./.github/workflows/pre-merge.yml + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] From 51910ee1b0fe3d8073ce85b443ede39aef9d7b3a Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:45:23 +0300 Subject: [PATCH 110/220] Use common from another repo --- .github/actions/apply-docker-version/action.yml | 7 +++++++ .../actions/apply-docker-version/apply-docker-version.sh | 5 +---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 315c56513..07cf7b3ec 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -14,6 +14,13 @@ outputs: runs: using: "composite" steps: + - name: Checkout common functions + uses: actions/checkout@v4 + with: + repository: redis/redis-oss-release-automation + ref: main + path: redis-oss-release-automation + - name: Apply docker version id: apply-version shell: bash diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 5ca21d469..243e645eb 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -13,11 +13,8 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" - +. "$GITHUB_WORKSPACE/redis-oss-release-automation/.github/actions/common/helpers.sh" # Input TAG is expected in $1 TAG="$1" From 9b9253d229baf5e17f0f9683b8034153f71c37d9 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:50:31 +0300 Subject: [PATCH 111/220] Fix remote action usage instead of local --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index bf8e71bbb..0924c7eb3 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -49,7 +49,7 @@ jobs: with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally@main + - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} From 59788b31d37eba43ddc1077e41fd37d00cd79fbe Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 18:11:22 +0300 Subject: [PATCH 112/220] Test merge back without changed files --- .github/workflows/release_build_and_test.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 26e7a46bf..b54ade886 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -20,8 +20,6 @@ run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format jobs: prepare-release: runs-on: ["ubuntu-latest"] - outputs: - changed_files: ${{ steps.apply-version.outputs.changed_files }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -48,14 +46,18 @@ jobs: build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} + # uses: ./.github/workflows/pre-merge.yml + # secrets: inherit + # with: + # release_tag: ${{ github.event.inputs.release_tag }} + runs-on: [ubuntu-latest] + steps: + - run: | + echo "Hello World" merge-back-to-release-branch: needs: [prepare-release, build-and-test] - if: success() && needs.prepare-release.outputs.changed_files != '' + if: success() runs-on: ["ubuntu-latest"] steps: - name: Checkout code @@ -74,4 +76,4 @@ jobs: with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} - gh_token: ${{ secrets.GITHUB_TOKEN }} + gh_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From d3f4cb661bd6699424f0e13115090ad6bd88d2d9 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 18:24:59 +0300 Subject: [PATCH 113/220] Return full build --- .github/workflows/release_build_and_test.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b54ade886..3cf591b98 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -46,14 +46,11 @@ jobs: build-and-test: needs: prepare-release - # uses: ./.github/workflows/pre-merge.yml - # secrets: inherit - # with: - # release_tag: ${{ github.event.inputs.release_tag }} - runs-on: [ubuntu-latest] - steps: - - run: | - echo "Hello World" + uses: ./.github/workflows/pre-merge.yml + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} + merge-back-to-release-branch: needs: [prepare-release, build-and-test] From a531b37d0d9c0a68d57b30b65190f41320a604d1 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Tue, 9 Sep 2025 21:02:08 +0300 Subject: [PATCH 114/220] Prepare stackbrew librirary intermediate commit --- .../apply-docker-version.sh | 7 +- .github/actions/common/func.sh | 269 ++++++++++++++++++ .../generate-stackbrew-library.sh | 54 ++++ test/run-shell-func-tests.sh | 246 ++++++++++++++++ 4 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 .github/actions/common/func.sh create mode 100755 .github/actions/create-library-pr/generate-stackbrew-library.sh create mode 100755 test/run-shell-func-tests.sh diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 243e645eb..0078b05cb 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -13,8 +13,13 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$GITHUB_WORKSPACE/redis-oss-release-automation/.github/actions/common/helpers.sh" +. "$SCRIPT_DIR/../common/func.sh" + +source_helper_file helpers.sh # Input TAG is expected in $1 TAG="$1" diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh new file mode 100644 index 000000000..d0afa9d0e --- /dev/null +++ b/.github/actions/common/func.sh @@ -0,0 +1,269 @@ +#!/bin/bash + +source_helper_file() { + local helper_file="$1" + local helper_errors="" + for dir in "GITHUB_WORKSPACE:$GITHUB_WORKSPACE/redis-oss-release-automation" "RELEASE_AUTOMATION_DIR:$RELEASE_AUTOMATION_DIR" ":../redis-oss-release-automation"; do + local var_name="${dir%%:*}" + local dir="${dir#*:}" + if [ -n "$var_name" ]; then + var_name="\$$var_name" + fi + local helper_path="$dir/.github/actions/common/$helper_file" + if [ -f "$helper_path" ]; then + helper_errors="" + # shellcheck disable=SC1090 + . "$helper_path" + break + else + helper_errors=$(printf "%s\n %s: %s" "$helper_errors" "$var_name" "$helper_path") + fi + done + if [ -n "$helper_errors" ]; then + echo "Error: $helper_file not found in any of the following locations: $helper_errors" >&2 + exit 1 + fi +} + +git_ls_remote_major_release_version_branches() { + local remote="$1" + local major_version="$2" + execute_command --no-std -- git ls-remote --heads "$remote" "release/$major_version.*" + if [ -z "$last_cmd_stdout" ]; then + console_output 0 red "Error: No release branches found for major_version=$major_version" + return 1 + fi + echo "$last_cmd_stdout" +} + +git_ls_remote_tags() { + local remote="$1" + local major_version="$2" + execute_command git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" +} + +filter_major_release_version_branches() { + local major_version="$1" + while read -r line; do + local ref="$(echo "$line" | awk '{print $2}')" + local commit="$(echo "$line" | awk '{print $1}')" + if echo "$ref" | grep -q "release/$major_version\.[0-9][0-9]*$"; then + echo "$ref $commit" + fi + done | sort -Vr +} + +filter_actual_major_redis_versions() { + local major_version="$1" + local last_minor="" last_is_milestone="" + local ref commit version_tag + + while read -r commit ref; do + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + echo "$version_tag $ref $commit" + done | sort -Vr | while read -r version_tag ref commit; do + local major minor patch suffix is_milestone + IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") + + if [ -n "$suffix" ]; then + is_milestone=1 + else + is_milestone="" + fi + + if [ "$last_minor" = "$minor" ] && [ "$last_is_milestone" = "$is_milestone" ]; then + console_output 2 gray "Skipping $version_tag, already have minor=$last_minor is_milestone=$last_is_milestone" + continue + fi + last_minor="$minor" + last_is_milestone="$is_milestone" + + console_output 2 gray "$version_tag $commit" + echo "$version_tag $commit" + done +} + +get_major_release_version_branches () { + local remote="$1" + local major_version="$2" + execute_command git_ls_remote_major_release_version_branches "$remote" "$major_version" | execute_command filter_major_release_version_branches "$major_version" +} + +get_actual_major_redis_versions() { + local remote="$1" + local major_version="$2" + execute_command git_ls_remote_tags "$remote" "$major_version" | execute_command filter_actual_major_redis_versions "$major_version" +} + +git_fetch_unshallow_refs() { + local remote="$1" + local refs_to_fetch + while read -r line; do + local ref="$(echo "$line" | awk '{print $1}')" + refs_to_fetch="$refs_to_fetch $ref" + done + # shellcheck disable=SC2086 + execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch +} + +extract_distro_name_from_dockerfile() { + local base_img + base_img="$(grep -m1 -i '^from' | awk '{print $2}')" + + increase_indent_level + console_output 2 gray "Extracting distro from dockerfile" + + if echo "$base_img" | grep -q 'alpine:'; then + distro="$(echo "$base_img" | tr -d ':')" + elif echo "$base_img" | grep -q 'debian:'; then + distro="$(echo "${base_img//-slim/}" | awk -F: '{print $2}')" + else + console_output 0 red "Error: Unknown base image $base_img" + decrease_indent_level + return 1 + fi + console_output 2 gray "distro=$distro" + decrease_indent_level + echo "$distro" +} + +extract_redis_version_from_dockerfile() { + increase_indent_level + console_output 2 gray "Extracting redis version from dockerfile" + local redis_version + redis_version=$(grep -m1 -i '^ENV REDIS_DOWNLOAD_URL.*https*:.*tar' \ + | sed 's/ENV.*REDIS_DOWNLOAD_URL.*[-/]\([1-9][0-9]*\..*\)\.tar\.gz/\1/g' \ + | grep -E '^[1-9][0-9]*\.' + ) + console_output 2 gray "redis_version=$redis_version" + if [ -z "$redis_version" ]; then + console_output 0 red "Error: Failed to extract redis version from dockerfile" + decrease_indent_level + return 1 + fi + echo "$redis_version" + decrease_indent_level +} + +redis_version_split() { + local version + local numerics + # shellcheck disable=SC2001 + version=$(echo "$1" | sed 's/^v//') + + numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)') + if [ -z "$numerics" ]; then + console_output 2 red "Cannot split version '$version', incorrect version format" + return 1 + fi + local major minor patch suffix + IFS=. read -r major minor patch < <(echo "$numerics") + suffix=${version:${#numerics}} + printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" +} + + +git_show_file_from_ref() { + local ref=$1 + local file=$2 + execute_command git show "$ref:$file" +} + +generate_tags_list() { + local redis_version=$1 + local distro_names=$2 + local is_latest=$3 + local is_default=$4 + + local tags versions + + local major minor patch suffix + IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") + + local mainline_version + mainline_version="$major.$minor" + + versions=("$redis_version" "$mainline_version") + if [ "$is_latest" = 1 ]; then + versions+=("$major") + fi + + if [ "$is_default" = 1 ]; then + tags=("${versions[@]}") + fi + + for distro_name in $distro_names; do + for v in "${versions[@]}"; do + tags+=("$v-$distro_name") + done + done + + if [ "$is_latest" = 1 ]; then + if [ "$is_default" = 1 ]; then + tags+=("latest") + fi + # shellcheck disable=SC2206 + tags+=($distro_names) + fi + # shellcheck disable=SC2001 + echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" +} + +generate_stackbrew_library() { + local commit redis_version distro distro_version + local is_latest="unset" is_default + + local stackbrew_content + + while read -r commit redis_version distro distro_version; do + local major minor patch suffix + IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") + + # assigning latest to the first non milestone (empty suffix) version from top + if [ "$is_latest" = "unset" ]; then + if [ -z "$suffix" ]; then + is_latest=1 + fi + else + is_latest="" + fi + + if echo "$distro" | grep -q 'alpine'; then + is_default="" + distro_names="$distro $distro_version" + else + is_default=1 + distro_names="$distro_version" + fi + + local tags + tags=$(generate_tags_list "$redis_version" "$distro_names" "$is_latest" "$is_default") + printf -v stackbrew_content "%s%s\n" "$stackbrew_content" "$tags" + done + printf %s "$stackbrew_content" + console_output 2 gray "$stackbrew_content" +} + +prepare_releases_list() { + local redis_version commit + local debug_output version_line + while read -r redis_version commit; do + for distro in debian alpine; do + local dockerfile distro_version redis_version + dockerfile=$(git_show_file_from_ref "$commit" "$distro/Dockerfile") + console_output 3 gray "$dockerfile" + + distro_version=$(echo "$dockerfile" | extract_distro_name_from_dockerfile) + # validate version + redis_version_split "$redis_version" >/dev/null + + printf -v version_line "%s %s %s %s\n" "$commit" "$redis_version" "$distro" "$distro_version" + printf "%s" "$version_line" + printf -v debug_output "%s%s" "$debug_output" "$version_line" + done + done + console_output 2 gray "Final Releases list:" + increase_indent_level + console_output 2 gray "$debug_output" + decrease_indent_level +} \ No newline at end of file diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh new file mode 100755 index 000000000..011ec7914 --- /dev/null +++ b/.github/actions/create-library-pr/generate-stackbrew-library.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +if [ -z "$VERBOSITY" ]; then + VERBOSITY=1 +fi + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/func.sh" + +source_helper_file helpers.sh +source_helper_file github_helpers.sh + +init_console_output + +MAJOR_VERSION="" +REMOTE="origin" +while [[ $# -gt 0 ]]; do + case $1 in + --major-version) + MAJOR_VERSION=$2 + shift + shift + ;; + --remote) + REMOTE=$2 + shift + shift + ;; + *) + echo "Error: Unknown option $1" + exit 1 + ;; + esac +done + +if [ -z "$MAJOR_VERSION" ]; then + echo "Error: --major-version M is required as argument" + exit 1 +fi + +execute_command git fetch +redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") +echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" +echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library + diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh new file mode 100755 index 000000000..ef7e0746f --- /dev/null +++ b/test/run-shell-func-tests.sh @@ -0,0 +1,246 @@ +#!/bin/bash +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../.github/actions/common/func.sh" + +source_helper_file "helpers.sh" + +test_get_distro_name_from_dockerfile() { + distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) + assertEquals "alpine3.22" "$distro_name" + + distro_name=$(echo 'FROM debian:bookworm-slim' | extract_distro_name_from_dockerfile) + assertEquals "bookworm" "$distro_name" + + distro_name=$(echo 'FROM debian:bookworm' | extract_distro_name_from_dockerfile) + assertEquals "bookworm" "$distro_name" +} + +test_filter_major_release_version_branches() { + local input + input=$(cat </dev/null) + local ret=$? + assertNotEquals "extract_redis_version_from_dockerfile returned non zero" "0" "$ret" +} + +test_generate_tags_list() { + local tags + tags=$(generate_tags_list 8.2.1 bookworm 1 1) + assertEquals "8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm" "$tags" + + tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 0) + assertEquals "8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22" "$tags" +} + +test_redis_version_split() { + local major minor patch suffix + local version + + version="8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="v8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="v8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" + + version="v8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" +} + +test_filter_actual_major_release_version() { + version=$(cat < Date: Wed, 10 Sep 2025 14:57:20 +0300 Subject: [PATCH 115/220] Support for EOLing versions, fixes and update tests --- .github/actions/common/func.sh | 97 ++++++++++---- test/run-shell-func-tests.sh | 226 +++++++++++++++++++++------------ 2 files changed, 212 insertions(+), 111 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index d0afa9d0e..7d548247e 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -39,7 +39,12 @@ git_ls_remote_major_release_version_branches() { git_ls_remote_tags() { local remote="$1" local major_version="$2" - execute_command git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" + execute_command --no-std -- git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" + if [ -z "$last_cmd_stdout" ]; then + console_output 0 red "Error: No tags found for major_version=$major_version" + return 1 + fi + echo "$last_cmd_stdout" } filter_major_release_version_branches() { @@ -53,6 +58,51 @@ filter_major_release_version_branches() { done | sort -Vr } +sort_version_tags() { + local major_version="$1" + local version_tag commit ref + while read -r commit ref; do + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + printf "%s %s\n" "$version_tag" "$commit" + done | sort -Vr +} + +filter_out_eol_versions() { + local major_version="$1" + local version_tag commit + local last_minor skip_minor minors + local major minor patch suffix + local versions + + mapfile -t versions + for line in "${versions[@]}"; do + read -r version_tag commit < <(echo "$line") + IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") + + if [ "$minor" != "$last_minor" ] && [ -n "$last_minor" ]; then + if [ -z "$skip_minor" ]; then + printf "%s" "$minors" + else + console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" + fi + minors="" + skip_minor="" + fi + last_minor="$minor" + + printf -v minors "%s%s\n" "$minors" "$version_tag $commit" + + if echo "$suffix" | grep -qi "-eol$"; then + skip_minor="$minor" + fi + done + if [ -z "$skip_minor" ]; then + printf "%s" "$minors" + else + console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" + fi +} + filter_actual_major_redis_versions() { local major_version="$1" local last_minor="" last_is_milestone="" @@ -127,24 +177,6 @@ extract_distro_name_from_dockerfile() { echo "$distro" } -extract_redis_version_from_dockerfile() { - increase_indent_level - console_output 2 gray "Extracting redis version from dockerfile" - local redis_version - redis_version=$(grep -m1 -i '^ENV REDIS_DOWNLOAD_URL.*https*:.*tar' \ - | sed 's/ENV.*REDIS_DOWNLOAD_URL.*[-/]\([1-9][0-9]*\..*\)\.tar\.gz/\1/g' \ - | grep -E '^[1-9][0-9]*\.' - ) - console_output 2 gray "redis_version=$redis_version" - if [ -z "$redis_version" ]; then - console_output 0 red "Error: Failed to extract redis version from dockerfile" - decrease_indent_level - return 1 - fi - echo "$redis_version" - decrease_indent_level -} - redis_version_split() { local version local numerics @@ -175,6 +207,8 @@ generate_tags_list() { local is_latest=$3 local is_default=$4 + console_output 2 gray "Generate tags redis_version=$redis_version distro_names=$distro_names is_latest=$is_latest id_default=$is_default" + local tags versions local major minor patch suffix @@ -183,8 +217,13 @@ generate_tags_list() { local mainline_version mainline_version="$major.$minor" - versions=("$redis_version" "$mainline_version") - if [ "$is_latest" = 1 ]; then + versions=("$redis_version") + # generate mainline version tag only for GA releases, e.g 8.2 and 8.2-distro + # tags will be generated only for 8.2.1 but not for 8.2.1-m01 + if [ -z "$suffix" ]; then + versions+=("$mainline_version") + fi + if [ "$is_latest" != "" ]; then versions+=("$major") fi @@ -198,7 +237,7 @@ generate_tags_list() { done done - if [ "$is_latest" = 1 ]; then + if [ "$is_latest" != "" ]; then if [ "$is_default" = 1 ]; then tags+=("latest") fi @@ -211,20 +250,24 @@ generate_tags_list() { generate_stackbrew_library() { local commit redis_version distro distro_version - local is_latest="unset" is_default + local is_latest="" is_latest_unset=1 is_default local stackbrew_content - while read -r commit redis_version distro distro_version; do + mapfile -t releases + for line in "${releases[@]}"; do + read -r commit redis_version distro distro_version < <(echo "$line") + local major minor patch suffix IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") # assigning latest to the first non milestone (empty suffix) version from top - if [ "$is_latest" = "unset" ]; then + if [ "$is_latest_unset" = 1 ]; then if [ -z "$suffix" ]; then - is_latest=1 + is_latest="$minor" + is_latest_unset="" fi - else + elif [ "$is_latest" != "$minor" ]; then is_latest="" fi diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index ef7e0746f..008f25d5a 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -5,6 +5,8 @@ SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" source_helper_file "helpers.sh" +init_console_output + test_get_distro_name_from_dockerfile() { distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) assertEquals "alpine3.22" "$distro_name" @@ -44,77 +46,19 @@ EXPECTED assertEquals "$expected" "$output" } -test_extract_redis_version_from_dockerfile_github() { - local dockerfile_content - dockerfile_content=$(cat </dev/null) - local ret=$? - assertNotEquals "extract_redis_version_from_dockerfile returned non zero" "0" "$ret" -} - test_generate_tags_list() { local tags tags=$(generate_tags_list 8.2.1 bookworm 1 1) assertEquals "8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm" "$tags" - tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 0) + tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 "") assertEquals "8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22" "$tags" + + tags=$(generate_tags_list 8.0-m03 "bookworm" "" 1) + assertEquals "8.0-m03, 8.0-m03-bookworm" "$tags" + + tags=$(generate_tags_list 8.0-m03 "alpine alpine3.21" "" 1) + assertEquals "8.0-m03, 8.0-m03-alpine, 8.0-m03-alpine3.21" "$tags" } test_redis_version_split() { @@ -170,14 +114,24 @@ test_redis_version_split() { assertEquals "suffix of $version" "-m03-int" "$suffix" } +test_redis_version_split_fail() { + IFS=: read -r major minor patch suffix < <(redis_version_split 8.x.x) + assertNotEquals "return code" "0" "$?" +} + + test_filter_actual_major_release_version() { version=$(cat < Date: Thu, 11 Sep 2025 17:03:42 +0300 Subject: [PATCH 116/220] Comments and fixes to the shell version --- .github/actions/common/func.sh | 63 ++++++++++++------- .../generate-stackbrew-library.sh | 3 +- test/run-shell-func-tests.sh | 3 + 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 7d548247e..efb4ccee3 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Sources a helper file from multiple possible locations (GITHUB_WORKSPACE, RELEASE_AUTOMATION_DIR, or relative path) source_helper_file() { local helper_file="$1" local helper_errors="" @@ -25,6 +26,7 @@ source_helper_file() { fi } +# Lists remote release branches for a specific major version (e.g., release/8.*) git_ls_remote_major_release_version_branches() { local remote="$1" local major_version="$2" @@ -36,6 +38,7 @@ git_ls_remote_major_release_version_branches() { echo "$last_cmd_stdout" } +# Lists remote tags for a specific major version (e.g., v8.*) git_ls_remote_tags() { local remote="$1" local major_version="$2" @@ -47,6 +50,7 @@ git_ls_remote_tags() { echo "$last_cmd_stdout" } +# Filters and sorts release branches by major version in reverse version order filter_major_release_version_branches() { local major_version="$1" while read -r line; do @@ -58,19 +62,29 @@ filter_major_release_version_branches() { done | sort -Vr } +# Sorts version tags in reverse version order for a specific major version +# stdin: commit ref (git ls-remote) +# stdout: version commit (vX.X.X sha1) - sorted by version sort_version_tags() { local major_version="$1" local version_tag commit ref while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*" || :)" + if [ -z "$version_tag" ]; then + console_output 2 red "Incorrect reference format: $ref" + return 1 + fi printf "%s %s\n" "$version_tag" "$commit" done | sort -Vr } +# Filters out end-of-life (EOL) versions by skipping entire minor version series marked with -eol suffix +# stdin: version commit (vX.X.X sha1) - must be sorted by version +# stdout: version commit (vX.X.X sha1) filter_out_eol_versions() { local major_version="$1" local version_tag commit - local last_minor skip_minor minors + local last_minor="" skip_minor="" minors="" local major minor patch suffix local versions @@ -103,15 +117,15 @@ filter_out_eol_versions() { fi } +# Filters Redis versions to keep only the latest patch version (and optionally the latest milestone) for each minor version +# stdin: version commit (vX.X.X sha1) - must be sorted by version +# stdout: version commit (vX.X.X sha1) filter_actual_major_redis_versions() { local major_version="$1" local last_minor="" last_is_milestone="" local ref commit version_tag - - while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" - echo "$version_tag $ref $commit" - done | sort -Vr | while read -r version_tag ref commit; do + console_output 2 gray "filter_actual_major_redis_versions" + while read -r version_tag commit; do local major minor patch suffix is_milestone IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") @@ -133,21 +147,20 @@ filter_actual_major_redis_versions() { done } -get_major_release_version_branches () { - local remote="$1" - local major_version="$2" - execute_command git_ls_remote_major_release_version_branches "$remote" "$major_version" | execute_command filter_major_release_version_branches "$major_version" -} - +# Gets and filters actual Redis versions (tags) from a remote repository for a major version get_actual_major_redis_versions() { local remote="$1" local major_version="$2" - execute_command git_ls_remote_tags "$remote" "$major_version" | execute_command filter_actual_major_redis_versions "$major_version" + execute_command git_ls_remote_tags "$remote" "$major_version" \ + | execute_command sort_version_tags "$major_version" \ + | execute_command filter_out_eol_versions "$major_version" \ + | execute_command filter_actual_major_redis_versions "$major_version" } +# Fetches unshallow refs from a remote repository for the provided list of references git_fetch_unshallow_refs() { local remote="$1" - local refs_to_fetch + local refs_to_fetch="" while read -r line; do local ref="$(echo "$line" | awk '{print $1}')" refs_to_fetch="$refs_to_fetch $ref" @@ -156,6 +169,7 @@ git_fetch_unshallow_refs() { execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch } +# Extracts the distribution name from a Dockerfile's FROM statement (supports Alpine and Debian) extract_distro_name_from_dockerfile() { local base_img base_img="$(grep -m1 -i '^from' | awk '{print $2}')" @@ -177,13 +191,14 @@ extract_distro_name_from_dockerfile() { echo "$distro" } +# Splits a Redis version string into major:minor:patch:suffix components redis_version_split() { local version local numerics # shellcheck disable=SC2001 version=$(echo "$1" | sed 's/^v//') - numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)') + numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)' || :) if [ -z "$numerics" ]; then console_output 2 red "Cannot split version '$version', incorrect version format" return 1 @@ -194,21 +209,23 @@ redis_version_split() { printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" } - +# Shows a file from a specific git reference (commit/branch/tag) git_show_file_from_ref() { local ref=$1 local file=$2 execute_command git show "$ref:$file" } +# Generates a comma-separated list of Docker tags for a Redis version and distribution +# args: redis_version distro_names is_latest is_default +# is_latest empty for non-latest, otherwise latest +# is_default 1 for default distro, otherwise not default generate_tags_list() { local redis_version=$1 local distro_names=$2 local is_latest=$3 local is_default=$4 - console_output 2 gray "Generate tags redis_version=$redis_version distro_names=$distro_names is_latest=$is_latest id_default=$is_default" - local tags versions local major minor patch suffix @@ -248,11 +265,13 @@ generate_tags_list() { echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" } +# Generates stackbrew library content (for specific major version) +# stdin: commit redis_version distro distro_version (sha1 vX.X.X alpine alpine3.21) generate_stackbrew_library() { local commit redis_version distro distro_version local is_latest="" is_latest_unset=1 is_default - local stackbrew_content + local stackbrew_content="" mapfile -t releases for line in "${releases[@]}"; do @@ -287,9 +306,11 @@ generate_stackbrew_library() { console_output 2 gray "$stackbrew_content" } +# Prepares a list of releases with commit, Redis version, distro, and distro version information +# stdin: redis_version commit prepare_releases_list() { local redis_version commit - local debug_output version_line + local debug_output="" version_line while read -r redis_version commit; do for distro in debian alpine; do local dockerfile distro_version redis_version diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh index 011ec7914..ed1fffdcc 100755 --- a/.github/actions/create-library-pr/generate-stackbrew-library.sh +++ b/.github/actions/create-library-pr/generate-stackbrew-library.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -o pipefail # shellcheck disable=SC2034 last_cmd_stdout="" @@ -47,7 +48,7 @@ if [ -z "$MAJOR_VERSION" ]; then exit 1 fi -execute_command git fetch +set -u redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index 008f25d5a..1f4c98c94 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -1,10 +1,13 @@ #!/bin/bash +set -e -o pipefail SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../.github/actions/common/func.sh" source_helper_file "helpers.sh" +set -u + init_console_output test_get_distro_name_from_dockerfile() { From bbd49d2d9904b62c733f7a4464ad121e27f94e35 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:33:19 +0300 Subject: [PATCH 117/220] Python release-automation implementation --- .../workflows/build_release_automation.yml | 80 ++++++++ release-automation/README.md | 17 ++ release-automation/pyproject.toml | 86 ++++++++ .../src/stackbrew_generator/__init__.py | 3 + .../src/stackbrew_generator/cli.py | 139 +++++++++++++ .../src/stackbrew_generator/distribution.py | 117 +++++++++++ .../src/stackbrew_generator/exceptions.py | 138 +++++++++++++ .../src/stackbrew_generator/git_operations.py | 156 ++++++++++++++ .../src/stackbrew_generator/logging_config.py | 95 +++++++++ .../src/stackbrew_generator/models.py | 177 ++++++++++++++++ .../src/stackbrew_generator/stackbrew.py | 131 ++++++++++++ .../src/stackbrew_generator/version_filter.py | 153 ++++++++++++++ release-automation/tests/__init__.py | 1 + release-automation/tests/test_integration.py | 100 +++++++++ release-automation/tests/test_models.py | 179 ++++++++++++++++ release-automation/tests/test_stackbrew.py | 193 ++++++++++++++++++ 16 files changed, 1765 insertions(+) create mode 100644 .github/workflows/build_release_automation.yml create mode 100644 release-automation/README.md create mode 100644 release-automation/pyproject.toml create mode 100644 release-automation/src/stackbrew_generator/__init__.py create mode 100644 release-automation/src/stackbrew_generator/cli.py create mode 100644 release-automation/src/stackbrew_generator/distribution.py create mode 100644 release-automation/src/stackbrew_generator/exceptions.py create mode 100644 release-automation/src/stackbrew_generator/git_operations.py create mode 100644 release-automation/src/stackbrew_generator/logging_config.py create mode 100644 release-automation/src/stackbrew_generator/models.py create mode 100644 release-automation/src/stackbrew_generator/stackbrew.py create mode 100644 release-automation/src/stackbrew_generator/version_filter.py create mode 100644 release-automation/tests/__init__.py create mode 100644 release-automation/tests/test_integration.py create mode 100644 release-automation/tests/test_models.py create mode 100644 release-automation/tests/test_stackbrew.py diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml new file mode 100644 index 000000000..f20adf8eb --- /dev/null +++ b/.github/workflows/build_release_automation.yml @@ -0,0 +1,80 @@ +name: Build Release Automation Docker Image + +on: + workflow_dispatch: + inputs: + image_tag: + description: 'Docker image tag (default: latest)' + required: false + default: 'latest' + push_to_ghcr: + description: 'Push image to GHCR' + required: false + default: true + type: boolean + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/release-automation + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ github.event.inputs.image_tag }} + type=raw,value=latest,enable={{is_default_branch}} + type=sha,prefix={{branch}}- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./release-automation + file: ./release-automation/docker/Dockerfile + push: ${{ github.event.inputs.push_to_ghcr == 'true' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Output image details + run: | + echo "## Docker Image Built Successfully! 🐳" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Tags:**" >> $GITHUB_STEP_SUMMARY + echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/' | sed 's/$/`/' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [[ "${{ github.event.inputs.push_to_ghcr }}" == "true" ]]; then + echo "āœ… **Image pushed to GHCR**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "To pull the image:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + else + echo "ā„¹ļø **Image built locally only (not pushed)**" >> $GITHUB_STEP_SUMMARY + fi diff --git a/release-automation/README.md b/release-automation/README.md new file mode 100644 index 000000000..70aec8fce --- /dev/null +++ b/release-automation/README.md @@ -0,0 +1,17 @@ +# Release automation helper + +## Installation + +### From Source + +```bash +cd release-automation +pip install -e . +``` + +### Development Installation + +```bash +cd release-automation +pip install -e ".[dev]" +``` diff --git a/release-automation/pyproject.toml b/release-automation/pyproject.toml new file mode 100644 index 000000000..50714fc96 --- /dev/null +++ b/release-automation/pyproject.toml @@ -0,0 +1,86 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stackbrew-library-generator" +version = "0.1.0" +description = "Stackbrew Library Generator for Redis Docker Images" +authors = [ + {name = "Redis Team", email = "team@redis.io"}, +] +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "typer[all]>=0.9.0", + "rich>=13.0.0", + "pydantic>=2.0.0", + "packaging>=21.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "isort>=5.12.0", + "mypy>=1.0.0", + "pre-commit>=3.0.0", +] + +[project.scripts] +release-automation = "stackbrew_generator.cli:app" + +[project.urls] +Homepage = "https://github.com/redis/docker-library-redis" +Repository = "https://github.com/redis/docker-library-redis" +Issues = "https://github.com/redis/docker-library-redis/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/stackbrew_generator"] + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/README.md", + "/pyproject.toml", +] + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--cov=stackbrew_generator", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] diff --git a/release-automation/src/stackbrew_generator/__init__.py b/release-automation/src/stackbrew_generator/__init__.py new file mode 100644 index 000000000..38a1b1c6e --- /dev/null +++ b/release-automation/src/stackbrew_generator/__init__.py @@ -0,0 +1,3 @@ +"""Stackbrew Library Generator for Redis Docker Images.""" + +__version__ = "0.1.0" diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py new file mode 100644 index 000000000..cdc1721b4 --- /dev/null +++ b/release-automation/src/stackbrew_generator/cli.py @@ -0,0 +1,139 @@ +"""CLI interface for stackbrew library generator.""" + +import typer +from rich.console import Console +from rich.traceback import install + +from .distribution import DistributionDetector +from .exceptions import StackbrewGeneratorError +from .git_operations import GitClient +from .logging_config import setup_logging +from .stackbrew import StackbrewGenerator +from .version_filter import VersionFilter + +# Install rich traceback handler +install(show_locals=True) + +app = typer.Typer( + name="release-automation", + help="Generate stackbrew library content for Redis Docker images", + add_completion=False, +) + +# Console for logging and user messages (stderr) +console = Console(stderr=True) + + +@app.command() +def generate( + major_version: int = typer.Argument( + ..., + help="Redis major version to process (e.g., 8 for Redis 8.x)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show generated content without outputting to stdout" + ), +) -> None: + """Generate stackbrew library content for Redis Docker images. + + This command: + 1. Fetches Redis version tags from the specified remote + 2. Filters versions to remove EOL and select latest patches + 3. Extracts distribution information from Dockerfiles + 4. Generates appropriate Docker tags for each version/distribution + 5. Outputs stackbrew library content + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if verbose: + console.print(f"[bold blue]Stackbrew Library Generator[/bold blue]") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") + if dry_run: + console.print("[yellow]DRY RUN MODE - Generated content will be shown but not output to stdout[/yellow]") + + try: + # Initialize components + git_client = GitClient(remote=remote) + version_filter = VersionFilter(git_client) + distribution_detector = DistributionDetector(git_client) + stackbrew_generator = StackbrewGenerator() + + # Get actual Redis versions to process + versions = version_filter.get_actual_major_redis_versions(major_version) + + if not versions: + console.print(f"[red]No versions found for Redis {major_version}.x[/red]") + raise typer.Exit(1) + + # Fetch required refs + refs_to_fetch = [commit for _, commit in versions] + git_client.fetch_refs(refs_to_fetch) + + # Prepare releases list with distribution information + releases = distribution_detector.prepare_releases_list(versions) + + if not releases: + console.print("[red]No releases prepared[/red]") + raise typer.Exit(1) + + # Generate stackbrew library content + entries = stackbrew_generator.generate_stackbrew_library(releases) + output = stackbrew_generator.format_stackbrew_output(entries) + + if dry_run: + console.print(f"[yellow]DRY RUN: Would generate stackbrew library with {len(entries)} entries[/yellow]") + if verbose: + console.print("[yellow]Generated content:[/yellow]") + console.print(output) + else: + if output: + # Output the stackbrew library content + print(output) + + if verbose: + console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + else: + console.print("[yellow]No stackbrew content generated[/yellow]") + + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + + +@app.command() +def version() -> None: + """Show version information.""" + from . import __version__ + console.print(f"stackbrew-library-generator {__version__}") + + +if __name__ == "__main__": + app() diff --git a/release-automation/src/stackbrew_generator/distribution.py b/release-automation/src/stackbrew_generator/distribution.py new file mode 100644 index 000000000..187e8f521 --- /dev/null +++ b/release-automation/src/stackbrew_generator/distribution.py @@ -0,0 +1,117 @@ +"""Distribution detection from Dockerfiles.""" + +from typing import List, Tuple + +from rich.console import Console + +from .exceptions import DistributionError +from .git_operations import GitClient +from .models import Distribution, RedisVersion, Release + +console = Console(stderr=True) + + +class DistributionDetector: + """Detects distribution information from Dockerfiles.""" + + def __init__(self, git_client: GitClient): + """Initialize distribution detector. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def extract_distribution_from_dockerfile(self, dockerfile_content: str) -> Distribution: + """Extract distribution information from Dockerfile content. + + Args: + dockerfile_content: Content of the Dockerfile + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + # Find the FROM line + from_line = None + for line in dockerfile_content.split('\n'): + line = line.strip() + if line.upper().startswith('FROM '): + from_line = line + break + + if not from_line: + raise DistributionError("No FROM line found in Dockerfile") + + try: + return Distribution.from_dockerfile_line(from_line) + except ValueError as e: + raise DistributionError(f"Failed to parse distribution from FROM line: {e}") from e + + def get_distribution_for_commit(self, commit: str, distro_type: str) -> Distribution: + """Get distribution information for a specific commit and distro type. + + Args: + commit: Git commit hash + distro_type: Distribution type ("debian" or "alpine") + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + dockerfile_path = f"{distro_type}/Dockerfile" + + try: + dockerfile_content = self.git_client.show_file(commit, dockerfile_path) + console.print(f"[dim]Retrieved {dockerfile_path} from {commit[:8]}[/dim]") + + distribution = self.extract_distribution_from_dockerfile(dockerfile_content) + console.print(f"[dim]Detected distribution: {distribution.type.value} {distribution.name}[/dim]") + + return distribution + + except Exception as e: + raise DistributionError( + f"Failed to get distribution for {distro_type} from {commit}: {e}" + ) from e + + def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> List[Release]: + """Prepare list of releases with distribution information. + + Args: + versions: List of (RedisVersion, commit) tuples + + Returns: + List of Release objects with distribution information + """ + console.print("[blue]Preparing releases list with distribution information[/blue]") + + releases = [] + distro_types = ["debian", "alpine"] + + for version, commit in versions: + console.print(f"[dim]Processing [bold yellow]{version}[/bold yellow] - {commit[:8]}[/dim]") + + for distro_type in distro_types: + try: + distribution = self.get_distribution_for_commit(commit, distro_type) + + release = Release( + commit=commit, + version=version, + distribution=distribution + ) + + releases.append(release) + console.print(f"[dim] Added: {release.console_repr()}[/dim]", highlight=False) + + except DistributionError as e: + console.print(f"[yellow]Warning: Failed to process {distro_type} for {version}: {e}[/yellow]") + continue + + console.print(f"[green]Prepared {len(releases)} releases[/green]") + return releases diff --git a/release-automation/src/stackbrew_generator/exceptions.py b/release-automation/src/stackbrew_generator/exceptions.py new file mode 100644 index 000000000..f5b11b2a6 --- /dev/null +++ b/release-automation/src/stackbrew_generator/exceptions.py @@ -0,0 +1,138 @@ +"""Custom exceptions for stackbrew library generation.""" + +from typing import Optional, Any, Dict + + +class StackbrewGeneratorError(Exception): + """Base exception for stackbrew generator errors. + + Provides structured error information with context and suggestions. + """ + + def __init__( + self, + message: str, + context: Optional[Dict[str, Any]] = None, + suggestion: Optional[str] = None, + original_error: Optional[Exception] = None + ): + """Initialize error with context. + + Args: + message: Error message + context: Additional context information + suggestion: Suggested fix or next steps + original_error: Original exception that caused this error + """ + super().__init__(message) + self.context = context or {} + self.suggestion = suggestion + self.original_error = original_error + + def get_detailed_message(self) -> str: + """Get detailed error message with context and suggestions.""" + parts = [str(self)] + + if self.context: + parts.append("Context:") + for key, value in self.context.items(): + parts.append(f" {key}: {value}") + + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + + if self.original_error: + parts.append(f"Original error: {self.original_error}") + + return "\n".join(parts) + + +class GitOperationError(StackbrewGeneratorError): + """Exception raised for Git operation failures.""" + + def __init__( + self, + message: str, + command: Optional[str] = None, + exit_code: Optional[int] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if command: + context['command'] = command + if exit_code is not None: + context['exit_code'] = exit_code + + suggestion = kwargs.get('suggestion') + if not suggestion and command: + if 'ls-remote' in command: + suggestion = "Check that the remote repository exists and is accessible" + elif 'fetch' in command: + suggestion = "Ensure you have network access and proper Git credentials" + elif 'show' in command: + suggestion = "Verify that the commit exists and contains the requested file" + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class VersionParsingError(StackbrewGeneratorError): + """Exception raised for version parsing failures.""" + + def __init__(self, message: str, version_string: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if version_string: + context['version_string'] = version_string + + suggestion = kwargs.get('suggestion', + "Version should be in format 'X.Y.Z' or 'vX.Y.Z' with optional suffix") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class DistributionError(StackbrewGeneratorError): + """Exception raised for distribution detection failures.""" + + def __init__( + self, + message: str, + dockerfile_path: Optional[str] = None, + from_line: Optional[str] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if dockerfile_path: + context['dockerfile_path'] = dockerfile_path + if from_line: + context['from_line'] = from_line + + suggestion = kwargs.get('suggestion', + "Dockerfile should have a FROM line with supported base image (alpine:* or debian:*)") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class ValidationError(StackbrewGeneratorError): + """Exception raised for validation failures.""" + + def __init__(self, message: str, field: Optional[str] = None, value: Optional[Any] = None, **kwargs): + context = kwargs.get('context', {}) + if field: + context['field'] = field + if value is not None: + context['value'] = value + + super().__init__(message, context=context, **kwargs) + + +class ConfigurationError(StackbrewGeneratorError): + """Exception raised for configuration errors.""" + + def __init__(self, message: str, config_key: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if config_key: + context['config_key'] = config_key + + suggestion = kwargs.get('suggestion', + "Check your configuration and environment variables") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) diff --git a/release-automation/src/stackbrew_generator/git_operations.py b/release-automation/src/stackbrew_generator/git_operations.py new file mode 100644 index 000000000..14dab035c --- /dev/null +++ b/release-automation/src/stackbrew_generator/git_operations.py @@ -0,0 +1,156 @@ +"""Git operations for stackbrew library generation.""" + +import re +import subprocess +from typing import Dict, List, Tuple + +from rich.console import Console + +from .exceptions import GitOperationError +from .models import RedisVersion + +console = Console(stderr=True) + + +class GitClient: + """Client for Git operations.""" + + def __init__(self, remote: str = "origin"): + """Initialize Git client. + + Args: + remote: Git remote name to use + """ + self.remote = remote + + def _run_command(self, cmd: List[str], capture_output: bool = True) -> subprocess.CompletedProcess: + """Run a git command with error handling. + + Args: + cmd: Command and arguments to run + capture_output: Whether to capture stdout/stderr + + Returns: + CompletedProcess result + + Raises: + GitOperationError: If command fails + """ + try: + result = subprocess.run( + cmd, + capture_output=capture_output, + text=True, + check=True, + ) + return result + except subprocess.CalledProcessError as e: + error_msg = f"Git command failed: {' '.join(cmd)}" + if e.stderr: + error_msg += f"\nError: {e.stderr.strip()}" + raise GitOperationError(error_msg) from e + except FileNotFoundError as e: + raise GitOperationError("Git command not found. Is git installed?") from e + + def list_remote_tags(self, major_version: int) -> List[Tuple[str, str]]: + """List remote tags for a specific major version. + + Args: + major_version: Major version to filter tags for + + Returns: + List of (commit, tag_ref) tuples + + Raises: + GitOperationError: If no tags found or git operation fails + """ + console.print(f"[dim]Listing remote tags for v{major_version}.*[/dim]") + + cmd = [ + "git", "ls-remote", "--refs", "--tags", + self.remote, f"refs/tags/v{major_version}.*" + ] + + result = self._run_command(cmd) + + if not result.stdout.strip(): + raise GitOperationError(f"No tags found for major version {major_version}") + + tags = [] + for line in result.stdout.strip().split('\n'): + if line: + commit, ref = line.split('\t', 1) + tags.append((commit, ref)) + + console.print(f"[dim]Found {len(tags)} tags[/dim]") + return tags + + def fetch_refs(self, refs: List[str]) -> None: + """Fetch specific refs from remote. + + Args: + refs: List of refs to fetch + + Raises: + GitOperationError: If fetch operation fails + """ + if not refs: + return + + console.print(f"[dim]Fetching {len(refs)} refs[/dim]") + + # Use git fetch with unshallow to ensure we have full history + cmd = ["git", "fetch", "--unshallow", self.remote] + refs + + try: + self._run_command(cmd, capture_output=False) + except GitOperationError: + # If --unshallow fails (repo already unshallow), try without it + cmd = ["git", "fetch", self.remote] + refs + self._run_command(cmd, capture_output=False) + + def show_file(self, commit: str, file_path: str) -> str: + """Show file content from a specific commit. + + Args: + commit: Git commit hash + file_path: Path to file in repository + + Returns: + File content as string + + Raises: + GitOperationError: If file cannot be retrieved + """ + cmd = ["git", "show", f"{commit}:{file_path}"] + + try: + result = self._run_command(cmd) + return result.stdout + except GitOperationError as e: + raise GitOperationError(f"Failed to get {file_path} from {commit}: {e}") from e + + def extract_version_from_tag(self, tag_ref: str, major_version: int) -> RedisVersion: + """Extract Redis version from tag reference. + + Args: + tag_ref: Git tag reference (e.g., refs/tags/v8.2.1) + major_version: Expected major version for validation + + Returns: + Parsed RedisVersion + + Raises: + GitOperationError: If tag format is invalid + """ + # Extract version from tag reference + match = re.search(rf"v{major_version}\.\d+(?:\.\d+)?.*", tag_ref) + if not match: + raise GitOperationError(f"Invalid tag format: {tag_ref}") + + version_str = match.group(0) + + try: + return RedisVersion.parse(version_str) + except ValueError as e: + raise GitOperationError(f"Failed to parse version from {tag_ref}: {e}") from e diff --git a/release-automation/src/stackbrew_generator/logging_config.py b/release-automation/src/stackbrew_generator/logging_config.py new file mode 100644 index 000000000..b261e7ae7 --- /dev/null +++ b/release-automation/src/stackbrew_generator/logging_config.py @@ -0,0 +1,95 @@ +"""Logging configuration for stackbrew generator.""" + +import logging +from typing import Optional + +from rich.console import Console +from rich.logging import RichHandler + + +def setup_logging( + level: str = "INFO", + verbose: bool = False, + console: Optional[Console] = None +) -> logging.Logger: + """Set up logging configuration. + + Args: + level: Logging level (DEBUG, INFO, WARNING, ERROR) + verbose: Enable verbose logging + console: Rich console instance to use (should use stderr) + + Returns: + Configured logger instance + """ + if console is None: + # Create console that outputs to stderr + console = Console(stderr=True) + + # Determine log level + if verbose: + log_level = logging.DEBUG + else: + log_level = getattr(logging, level.upper(), logging.INFO) + + # Configure root logger + logging.basicConfig( + level=log_level, + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler( + console=console, + show_path=verbose, + show_time=verbose, + rich_tracebacks=True, + tracebacks_show_locals=verbose, + ) + ], + ) + + # Get logger for our package + logger = logging.getLogger("stackbrew_generator") + logger.setLevel(log_level) + + return logger + + +def get_logger(name: str) -> logging.Logger: + """Get a logger instance for a specific module. + + Args: + name: Logger name (usually __name__) + + Returns: + Logger instance + """ + return logging.getLogger(f"stackbrew_generator.{name}") + + +class LoggingMixin: + """Mixin class to add logging capabilities to other classes.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = get_logger(self.__class__.__name__) + + def log_debug(self, message: str, *args, **kwargs) -> None: + """Log debug message.""" + self.logger.debug(message, *args, **kwargs) + + def log_info(self, message: str, *args, **kwargs) -> None: + """Log info message.""" + self.logger.info(message, *args, **kwargs) + + def log_warning(self, message: str, *args, **kwargs) -> None: + """Log warning message.""" + self.logger.warning(message, *args, **kwargs) + + def log_error(self, message: str, *args, **kwargs) -> None: + """Log error message.""" + self.logger.error(message, *args, **kwargs) + + def log_exception(self, message: str, *args, **kwargs) -> None: + """Log exception with traceback.""" + self.logger.exception(message, *args, **kwargs) diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py new file mode 100644 index 000000000..599bbe9c9 --- /dev/null +++ b/release-automation/src/stackbrew_generator/models.py @@ -0,0 +1,177 @@ +"""Data models for stackbrew library generation.""" + +import re +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, Field, validator + + +class DistroType(str, Enum): + """Distribution type enumeration.""" + + ALPINE = "alpine" + DEBIAN = "debian" + + +class RedisVersion(BaseModel): + """Represents a parsed Redis version.""" + + major: int = Field(..., ge=1, description="Major version number") + minor: int = Field(..., ge=0, description="Minor version number") + patch: Optional[int] = Field(None, ge=0, description="Patch version number") + suffix: str = Field("", description="Version suffix (e.g., -m01, -rc1, -eol)") + + @classmethod + def parse(cls, version_str: str) -> "RedisVersion": + """Parse a version string into components. + + Args: + version_str: Version string (e.g., "v8.2.1-m01", "8.2", "7.4.0-eol") + + Returns: + RedisVersion instance + + Raises: + ValueError: If version string format is invalid + """ + # Remove 'v' prefix if present + version = version_str.lstrip("v") + + # Extract numeric part and suffix + match = re.match(r"^([1-9]\d*\.\d+(?:\.\d+)?)(.*)", version) + if not match: + raise ValueError(f"Invalid version format: {version_str}") + + numeric_part, suffix = match.groups() + + # Parse numeric components + parts = numeric_part.split(".") + major = int(parts[0]) + minor = int(parts[1]) + patch = int(parts[2]) if len(parts) > 2 else None + + return cls(major=major, minor=minor, patch=patch, suffix=suffix) + + @property + def is_milestone(self) -> bool: + """Check if this is a milestone version (has suffix).""" + return bool(self.suffix) + + @property + def is_eol(self) -> bool: + """Check if this version is end-of-life.""" + return self.suffix.lower().endswith("-eol") + + @property + def mainline_version(self) -> str: + """Get the mainline version string (major.minor).""" + return f"{self.major}.{self.minor}" + + def __str__(self) -> str: + """String representation of the version.""" + version = f"{self.major}.{self.minor}" + if self.patch is not None: + version += f".{self.patch}" + return version + self.suffix + + def __lt__(self, other: "RedisVersion") -> bool: + """Compare versions for sorting.""" + if not isinstance(other, RedisVersion): + return NotImplemented + + # Compare major.minor.patch first + self_tuple = (self.major, self.minor, self.patch or 0) + other_tuple = (other.major, other.minor, other.patch or 0) + + if self_tuple != other_tuple: + return self_tuple < other_tuple + + # If numeric parts are equal, compare suffixes + # Empty suffix (GA) comes after suffixes (milestones) + if not self.suffix and other.suffix: + return False + if self.suffix and not other.suffix: + return True + + return self.suffix < other.suffix + + +class Distribution(BaseModel): + """Represents a Linux distribution.""" + + type: DistroType = Field(..., description="Distribution type") + name: str = Field(..., description="Distribution name/version") + + @classmethod + def from_dockerfile_line(cls, from_line: str) -> "Distribution": + """Parse distribution from Dockerfile FROM line. + + Args: + from_line: FROM line from Dockerfile (e.g., "FROM alpine:3.22") + + Returns: + Distribution instance + + Raises: + ValueError: If FROM line format is not supported + """ + # Extract base image from FROM line + parts = from_line.strip().split() + if len(parts) < 2 or parts[0].upper() != "FROM": + raise ValueError(f"Invalid FROM line: {from_line}") + + base_img = parts[1] + + if "alpine:" in base_img: + # Extract alpine version (e.g., alpine:3.22 -> alpine3.22) + version = base_img.split(":", 1)[1] + return cls(type=DistroType.ALPINE, name=f"alpine{version}") + elif "debian:" in base_img: + # Extract debian version, remove -slim suffix + version = base_img.split(":", 1)[1].replace("-slim", "") + return cls(type=DistroType.DEBIAN, name=version) + else: + raise ValueError(f"Unsupported base image: {base_img}") + + @property + def is_default(self) -> bool: + """Check if this is the default distribution (Debian).""" + return self.type == DistroType.DEBIAN + + @property + def tag_names(self) -> List[str]: + """Get tag name components for this distribution.""" + if self.type == DistroType.ALPINE: + return [self.type.value, self.name] + else: + return [self.name] + + +class Release(BaseModel): + """Represents a Redis release with distribution information.""" + + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + + def __str__(self) -> str: + """String representation of the release.""" + return f"{self.commit[:8]} {self.version} {self.distribution.type.value} {self.distribution.name}" + + def console_repr(self) -> str: + """Rich console representation with markup.""" + return f"{self.commit[:8]} [bold yellow]{self.version}[/bold yellow] {self.distribution.type.value} [bold yellow]{self.distribution.name}[/bold yellow]" + + +class StackbrewEntry(BaseModel): + """Represents a stackbrew library entry with tags.""" + + tags: List[str] = Field(..., description="Docker tags for this entry") + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + + def __str__(self) -> str: + """String representation as comma-separated tags.""" + return ", ".join(self.tags) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py new file mode 100644 index 000000000..e40bd4a6a --- /dev/null +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -0,0 +1,131 @@ +"""Stackbrew library generation.""" + +from typing import List + +from rich.console import Console + +from .models import Release, StackbrewEntry + +console = Console(stderr=True) + + +class StackbrewGenerator: + """Generates stackbrew library content.""" + + def generate_tags_for_release( + self, + release: Release, + is_latest: bool = False + ) -> List[str]: + """Generate Docker tags for a release. + + Args: + release: Release to generate tags for + is_latest: Whether this is the latest version + + Returns: + List of Docker tags + """ + tags = [] + version = release.version + distribution = release.distribution + + # Base version tags + version_tags = [str(version)] + + # Add mainline version tag only for GA releases (no suffix) + if not version.is_milestone: + version_tags.append(version.mainline_version) + + # Add major version tag for latest versions + if is_latest: + version_tags.append(str(version.major)) + + # For default distribution (Debian), add version tags without distro suffix + if distribution.is_default: + tags.extend(version_tags) + + # Add distro-specific tags + for distro_name in distribution.tag_names: + for version_tag in version_tags: + tags.append(f"{version_tag}-{distro_name}") + + # Add special latest tags + if is_latest: + if distribution.is_default: + tags.append("latest") + # Add bare distro names as tags + tags.extend(distribution.tag_names) + + return tags + + def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewEntry]: + """Generate stackbrew library entries from releases. + + Args: + releases: List of releases to process + + Returns: + List of StackbrewEntry objects + """ + console.print("[blue]Generating stackbrew library content[/blue]") + + if not releases: + console.print("[yellow]No releases to process[/yellow]") + return [] + + entries = [] + latest_minor = None + latest_minor_unset = True + + for release in releases: + # Determine latest version following bash logic: + # - Set latest_minor to the minor version of the first non-milestone version + # - Clear latest_minor if subsequent versions have different minor versions + if latest_minor_unset: + if not release.version.is_milestone: + latest_minor = release.version.minor + latest_minor_unset = False + console.print(f"[dim]Latest minor version set to: {latest_minor}[/dim]") + elif latest_minor != release.version.minor: + latest_minor = None + + # Check if this release should get latest tags + is_latest = latest_minor is not None + + # Generate tags for this release + tags = self.generate_tags_for_release(release, is_latest) + + if tags: + entry = StackbrewEntry( + tags=tags, + commit=release.commit, + version=release.version, + distribution=release.distribution + ) + entries.append(entry) + + console.print(f"[dim]{release.console_repr()} -> {len(tags)} tags[/dim]") + else: + console.print(f"[yellow]No tags generated for {release}[/yellow]") + + console.print(f"[green]Generated {len(entries)} stackbrew entries[/green]") + return entries + + def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: + """Format stackbrew entries as output string. + + Args: + entries: List of stackbrew entries + + Returns: + Formatted stackbrew library content + """ + if not entries: + return "" + + lines = [] + for entry in entries: + lines.append(str(entry)) + + return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py new file mode 100644 index 000000000..d928e0e83 --- /dev/null +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -0,0 +1,153 @@ +"""Version filtering and processing for Redis releases.""" + +from typing import Dict, List, Tuple + +from packaging.version import Version +from rich.console import Console + +from .git_operations import GitClient +from .models import RedisVersion + +console = Console(stderr=True) + + +class VersionFilter: + """Filters and processes Redis versions.""" + + def __init__(self, git_client: GitClient): + """Initialize version filter. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + """Get Redis versions from git tags. + + Args: + major_version: Major version to filter for + + Returns: + List of (RedisVersion, commit) tuples sorted by version (newest first) + """ + console.print(f"[blue]Getting Redis versions for major version {major_version}[/blue]") + + # Get remote tags + tags = self.git_client.list_remote_tags(major_version) + + # Parse versions from tags + versions = [] + for commit, tag_ref in tags: + try: + version = self.git_client.extract_version_from_tag(tag_ref, major_version) + versions.append((version, commit)) + except Exception as e: + console.print(f"[yellow]Warning: Skipping invalid tag {tag_ref}: {e}[/yellow]") + continue + + # Sort by version (newest first) + versions.sort(key=lambda x: x[0], reverse=True) + + console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") + return versions + + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + """Filter out end-of-life versions. + + Args: + versions: List of (RedisVersion, commit) tuples + + Returns: + Filtered list with EOL minor versions removed + """ + console.print("[blue]Filtering out EOL versions[/blue]") + + # Group versions by minor version + minor_versions: Dict[str, List[Tuple[RedisVersion, str]]] = {} + for version, commit in versions: + minor_key = version.mainline_version + if minor_key not in minor_versions: + minor_versions[minor_key] = [] + minor_versions[minor_key].append((version, commit)) + + # Check each minor version for EOL marker + filtered_versions = [] + for minor_key, minor_group in minor_versions.items(): + # Check if any version in this minor series is marked as EOL + has_eol = any(version.is_eol for version, _ in minor_group) + + if has_eol: + console.print(f"[yellow]Skipping minor version {minor_key}.* due to EOL[/yellow]") + else: + filtered_versions.extend(minor_group) + + # Sort again after filtering + filtered_versions.sort(key=lambda x: x[0], reverse=True) + + console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") + return filtered_versions + + def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + """Filter to keep only the latest patch version for each minor version and milestone status. + + Args: + versions: List of (RedisVersion, commit) tuples (should be sorted newest first) + + Returns: + Filtered list with only the latest versions for each minor/milestone combination + """ + console.print("[blue]Filtering to actual versions (latest patch per minor/milestone)[/blue]") + + seen_combinations = set() + filtered_versions = [] + + for version, commit in versions: + # Create a key for minor version + milestone status + combination_key = (version.mainline_version, version.is_milestone) + + if combination_key not in seen_combinations: + seen_combinations.add(combination_key) + filtered_versions.append((version, commit)) + + milestone_str = "milestone" if version.is_milestone else "GA" + console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") + else: + milestone_str = "milestone" if version.is_milestone else "GA" + console.print(f"[dim]Skipping {version} ({milestone_str}) - already have this minor/milestone combination[/dim]") + + console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") + return filtered_versions + + def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + """Get the actual Redis versions to process for a major version. + + This is the main entry point that combines all filtering steps: + 1. Get versions from git tags + 2. Filter out EOL versions + 3. Filter to actual versions (latest patch per minor/milestone) + + Args: + major_version: Major version to process + + Returns: + List of (RedisVersion, commit) tuples for processing + """ + console.print(f"[bold blue]Processing Redis {major_version}.x versions[/bold blue]") + + # Get all versions from tags + versions = self.get_redis_versions_from_tags(major_version) + + if not versions: + console.print(f"[red]No versions found for major version {major_version}[/red]") + return [] + + # Apply filters + versions = self.filter_eol_versions(versions) + versions = self.filter_actual_versions(versions) + + console.print(f"[green]Final selection: {len(versions)} versions to process[/green]") + for version, commit in versions: + console.print(f"[green] [bold yellow]{version}[/bold yellow] - {commit[:8]}[/green]") + + return versions diff --git a/release-automation/tests/__init__.py b/release-automation/tests/__init__.py new file mode 100644 index 000000000..a60351fa0 --- /dev/null +++ b/release-automation/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for stackbrew library generator.""" diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py new file mode 100644 index 000000000..066392eaf --- /dev/null +++ b/release-automation/tests/test_integration.py @@ -0,0 +1,100 @@ +"""Integration tests for the stackbrew generator.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType +from typer.testing import CliRunner + + +class TestIntegration: + """Integration tests for the complete workflow.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + @patch('stackbrew_generator.distribution.DistributionDetector') + @patch('stackbrew_generator.git_operations.GitClient') + def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distribution_detector_class): + """Test complete workflow in dry run mode.""" + # Mock git client + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + # Mock distribution detector + mock_distribution_detector = Mock() + mock_distribution_detector_class.return_value = mock_distribution_detector + + # Mock git operations + mock_git_client.list_remote_tags.return_value = [ + ("abc123", "refs/tags/v8.2.1"), + ("def456", "refs/tags/v8.2.0"), + ] + + mock_git_client.extract_version_from_tag.side_effect = [ + RedisVersion.parse("8.2.1"), + RedisVersion.parse("8.2.0"), + ] + + # Mock releases + from stackbrew_generator.models import Release, Distribution, DistroType + mock_releases = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + # Run command in dry run mode + result = self.runner.invoke(app, ["generate", "8", "--dry-run", "--verbose"]) + + # Check that it completed successfully + assert result.exit_code == 0 + assert "DRY RUN: Would generate stackbrew library" in result.stderr + assert "Generated content:" in result.stderr + + def test_version_command(self): + """Test version command.""" + result = self.runner.invoke(app, ["version"]) + assert result.exit_code == 0 + assert "stackbrew-library-generator" in result.stderr + + def test_invalid_major_version(self): + """Test handling of invalid major version.""" + result = self.runner.invoke(app, ["generate", "0"]) + assert result.exit_code != 0 + + @patch('stackbrew_generator.git_operations.GitClient') + def test_no_tags_found(self, mock_git_client_class): + """Test handling when no tags are found.""" + # Mock git client to return no tags + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + mock_git_client.list_remote_tags.return_value = [] + + result = self.runner.invoke(app, ["generate", "99"]) + assert result.exit_code == 1 + assert "No tags found" in result.stderr + + @patch('stackbrew_generator.version_filter.VersionFilter.get_actual_major_redis_versions') + def test_no_versions_found(self, mock_get_versions): + """Test handling when no versions are found.""" + # Mock git client to return no tags + mock_get_versions.return_value = [] + + result = self.runner.invoke(app, ["generate", "8"]) + #assert result.exit_code == 1 + assert "No versions found" in result.stderr + + def test_help_output(self): + """Test help output.""" + result = self.runner.invoke(app, ["generate", "--help"]) + assert result.exit_code == 0 + assert "Generate stackbrew library content" in result.stdout + assert "--remote" in result.stdout + assert "--verbose" in result.stdout + assert "--dry-run" in result.stdout diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py new file mode 100644 index 000000000..bc55fa9ab --- /dev/null +++ b/release-automation/tests/test_models.py @@ -0,0 +1,179 @@ +"""Tests for data models.""" + +import pytest + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release + + +class TestRedisVersion: + """Tests for RedisVersion model.""" + + def test_parse_basic_version(self): + """Test parsing basic version strings.""" + version = RedisVersion.parse("8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_v_prefix(self): + """Test parsing version with 'v' prefix.""" + version = RedisVersion.parse("v8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_suffix(self): + """Test parsing version with suffix.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "-m01" + + def test_parse_version_without_patch(self): + """Test parsing version without patch number.""" + version = RedisVersion.parse("8.2") + assert version.major == 8 + assert version.minor == 2 + assert version.patch is None + assert version.suffix == "" + + def test_parse_eol_version(self): + """Test parsing EOL version.""" + version = RedisVersion.parse("7.4.0-eol") + assert version.major == 7 + assert version.minor == 4 + assert version.patch == 0 + assert version.suffix == "-eol" + assert version.is_eol is True + + def test_parse_invalid_version(self): + """Test parsing invalid version strings.""" + with pytest.raises(ValueError): + RedisVersion.parse("invalid") + + with pytest.raises(ValueError): + RedisVersion.parse("0.1.0") # Major version must be >= 1 + + def test_is_milestone(self): + """Test milestone detection.""" + ga_version = RedisVersion.parse("8.2.1") + milestone_version = RedisVersion.parse("8.2.1-m01") + + assert ga_version.is_milestone is False + assert milestone_version.is_milestone is True + + def test_mainline_version(self): + """Test mainline version property.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.mainline_version == "8.2" + + def test_string_representation(self): + """Test string representation.""" + version1 = RedisVersion.parse("8.2.1") + version2 = RedisVersion.parse("8.2.1-m01") + version3 = RedisVersion.parse("8.2") + + assert str(version1) == "8.2.1" + assert str(version2) == "8.2.1-m01" + assert str(version3) == "8.2" + + def test_version_comparison(self): + """Test version comparison for sorting.""" + v1 = RedisVersion.parse("8.2.1") + v2 = RedisVersion.parse("8.2.2") + v3 = RedisVersion.parse("8.2.1-m01") + v4 = RedisVersion.parse("8.3.0") + + # Test numeric comparison + assert v1 < v2 + assert v2 < v4 + + # Test milestone vs GA (GA comes after milestone) + assert v3 < v1 + + # Test sorting + versions = [v4, v1, v3, v2] + sorted_versions = sorted(versions) + assert sorted_versions == [v3, v1, v2, v4] + + +class TestDistribution: + """Tests for Distribution model.""" + + def test_from_dockerfile_alpine(self): + """Test parsing Alpine distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM alpine:3.22") + assert distro.type == DistroType.ALPINE + assert distro.name == "alpine3.22" + + def test_from_dockerfile_debian(self): + """Test parsing Debian distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_debian_slim(self): + """Test parsing Debian slim distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm-slim") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_invalid(self): + """Test parsing invalid Dockerfile lines.""" + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("INVALID LINE") + + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("FROM unsupported:latest") + + def test_is_default(self): + """Test default distribution detection.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.is_default is False + assert debian.is_default is True + + def test_tag_names(self): + """Test tag name generation.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.tag_names == ["alpine", "alpine3.22"] + assert debian.tag_names == ["bookworm"] + + +class TestRelease: + """Tests for Release model.""" + + def test_release_creation(self): + """Test creating a Release instance.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution + ) + + assert release.commit == "abc123def456" + assert release.version == version + assert release.distribution == distribution + + def test_release_string_representation(self): + """Test Release string representation.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution + ) + + expected = "abc123de 8.2.1 debian bookworm" + assert str(release) == expected diff --git a/release-automation/tests/test_stackbrew.py b/release-automation/tests/test_stackbrew.py new file mode 100644 index 000000000..2d065513e --- /dev/null +++ b/release-automation/tests/test_stackbrew.py @@ -0,0 +1,193 @@ +"""Tests for stackbrew library generation.""" + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release +from stackbrew_generator.stackbrew import StackbrewGenerator + + +class TestStackbrewGenerator: + """Tests for StackbrewGenerator.""" + + def setup_method(self): + """Set up test fixtures.""" + self.generator = StackbrewGenerator() + + def test_generate_tags_debian_ga_latest(self): + """Test tag generation for Debian GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1", # Full version + "8.2", # Mainline version (GA only) + "8", # Major version (latest only) + "8.2.1-bookworm", # Version with distro + "8.2-bookworm", # Mainline with distro + "8-bookworm", # Major with distro + "latest", # Latest tag (default distro only) + "bookworm" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_debian_ga_not_latest(self): + """Test tag generation for Debian GA version (not latest).""" + version = RedisVersion.parse("7.4.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + expected_tags = [ + "7.4.1", # Full version + "7.4", # Mainline version (GA only) + "7.4.1-bookworm", # Version with distro + "7.4-bookworm" # Mainline with distro + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_alpine_ga_latest(self): + """Test tag generation for Alpine GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1-alpine", # Version with distro type + "8.2.1-alpine3.22", # Version with full distro name + "8.2-alpine", # Mainline with distro type + "8.2-alpine3.22", # Mainline with full distro name + "8-alpine", # Major with distro type + "8-alpine3.22", # Major with full distro name + "alpine", # Bare distro type (latest only) + "alpine3.22" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_milestone_version(self): + """Test tag generation for milestone version.""" + version = RedisVersion.parse("8.2.1-m01") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + # Milestone versions should not get mainline version tags or major version tags + expected_tags = [ + "8.2.1-m01", # Full version only + "8.2.1-m01-bookworm", # Version with distro + ] + + assert set(tags) == set(expected_tags) + + + + def test_generate_stackbrew_library(self): + """Test complete stackbrew library generation.""" + releases = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ), + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22") + ), + Release( + commit="def456", + version=RedisVersion.parse("8.1.5"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + assert len(entries) == 3 + + # Check that the 8.2.1 versions are marked as latest + debian_8_2_1 = next(e for e in entries if e.version.patch == 1 and e.distribution.type == DistroType.DEBIAN) + assert "latest" in debian_8_2_1.tags + assert "8" in debian_8_2_1.tags + + # Check that 8.1.5 is not marked as latest + debian_8_1_5 = next(e for e in entries if e.version.minor == 1) + assert "latest" not in debian_8_1_5.tags + assert "8" not in debian_8_1_5.tags + + def test_format_stackbrew_output(self): + """Test stackbrew output formatting.""" + entries = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + + stackbrew_entries = self.generator.generate_stackbrew_library(entries) + output = self.generator.format_stackbrew_output(stackbrew_entries) + + assert isinstance(output, str) + assert len(output) > 0 + # Should contain comma-separated tags + assert "," in output + + def test_generate_stackbrew_library_with_head_milestone(self): + """Test stackbrew generation with milestone at head (matches bash test).""" + # This matches the bash test case: test_generate_stackbrew_library_with_head_milestone + releases = [ + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.21") + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + # Expected tags based on bash test + expected_tags = [ + ["8.2.2-m01-int1", "8.2.2-m01-int1-bookworm"], # milestone - no major/mainline tags + ["8.2.2-m01-int1-alpine", "8.2.2-m01-int1-alpine3.22"], # milestone - no major/mainline tags + ["8.2.1", "8.2", "8", "8.2.1-bookworm", "8.2-bookworm", "8-bookworm", "latest", "bookworm"], # GA - gets all tags + ["8.2.1-alpine", "8.2-alpine", "8-alpine", "8.2.1-alpine3.22", "8.2-alpine3.22", "8-alpine3.22", "alpine", "alpine3.22"], # GA - gets all tags + ["8.0.3", "8.0", "8.0.3-bookworm", "8.0-bookworm"], # different minor - no major tags + ["8.0.3-alpine", "8.0-alpine", "8.0.3-alpine3.21", "8.0-alpine3.21"] # different minor - no major tags + ] + + assert len(entries) == 6 + for i, entry in enumerate(entries): + assert set(entry.tags) == set(expected_tags[i]), f"Tags mismatch for entry {i}: {entry.tags} != {expected_tags[i]}" From c06c3452f123abf6c243c8cb5472b4dce583efba Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:44:08 +0300 Subject: [PATCH 118/220] Triggering workflow --- .github/workflows/release_build_and_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3cf591b98..6715153a9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,6 +13,9 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false + push: + branches: + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 672c2aeb0b3a88557a6b6bd8f81f7cff921dc98c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:46:33 +0300 Subject: [PATCH 119/220] Fix trigger --- .github/workflows/build_release_automation.yml | 3 +++ .github/workflows/release_build_and_test.yml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index f20adf8eb..d07da428f 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,6 +12,9 @@ on: required: false default: true type: boolean + push: + branches: + - release/** env: REGISTRY: ghcr.io diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6715153a9..3cf591b98 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,9 +13,6 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From f5d5d0c23fdee29e5064b7e9264cedfa10e8f349 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:49:55 +0300 Subject: [PATCH 120/220] Docker for releasr-automation --- .github/workflows/build_release_automation.yml | 3 --- release-automation/docker/Dockerfile | 11 +++++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 release-automation/docker/Dockerfile diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index d07da428f..f20adf8eb 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,9 +12,6 @@ on: required: false default: true type: boolean - push: - branches: - - release/** env: REGISTRY: ghcr.io diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile new file mode 100644 index 000000000..653200a2a --- /dev/null +++ b/release-automation/docker/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-slim-trixie + +RUN apt update && apt -y install git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +COPY . /release-automation +RUN pip install -e /release-automation + +ENTRYPOINT ["release-automation"] +CMD ["--help"] From 1501da1408156670f74b570cefeda343ee52f446 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:56:39 +0300 Subject: [PATCH 121/220] Test release-automation docker --- .github/workflows/release_build_and_test.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3cf591b98..a84a86843 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,26 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Test Release Automation Docker Image + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ + -e GITHUB_ACTOR="${{ github.actor }}" \ + -e GITHUB_REPOSITORY="${{ github.repository }}" \ + ghcr.io/${{ github.repository }}/release-automation:latest \ + generate 8 + - name: Validate Redis Release Archive uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: From 13648c511477bcb79e1b5a91f0992d40e4d03fb7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:59:38 +0300 Subject: [PATCH 122/220] Fix image url --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a84a86843..3de2cd658 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -41,7 +41,7 @@ jobs: -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ -e GITHUB_ACTOR="${{ github.actor }}" \ -e GITHUB_REPOSITORY="${{ github.repository }}" \ - ghcr.io/${{ github.repository }}/release-automation:latest \ + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 - name: Validate Redis Release Archive From 3c5896d366d28238190d9b012fa0ebc7b1b59828 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:07:07 +0300 Subject: [PATCH 123/220] Test remote names --- .github/workflows/release_build_and_test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3de2cd658..b0c98b0ad 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,6 +35,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ + -e GITHUB_ACTOR="${{ github.actor }}" \ + -e GITHUB_REPOSITORY="${{ github.repository }}" \ + --entrypoint /bin/bash + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ + -c 'git remote -v' + docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ From 5b2eb4c66c27c1675bf3179482159d8e85246222 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:08:37 +0300 Subject: [PATCH 124/220] fix --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b0c98b0ad..ef1963a63 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -41,7 +41,7 @@ jobs: -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ -e GITHUB_ACTOR="${{ github.actor }}" \ -e GITHUB_REPOSITORY="${{ github.repository }}" \ - --entrypoint /bin/bash + --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ -c 'git remote -v' From ee0636fb40b346132b222bba728bd773d15b9525 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:10:24 +0300 Subject: [PATCH 125/220] fix add to safe directory --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ef1963a63..48a324222 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'git remote -v' + -c 'git config --global --add safe.directory /workspace && git remote -v' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From 555d12472d34cdd011826746f0270922d1cacb2e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:14:13 +0300 Subject: [PATCH 126/220] Debug output --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 48a324222..d164c7338 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'git config --global --add safe.directory /workspace && git remote -v' + -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From de932b0d672a9b375003b8b2a6c9b5024f6c5e69 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:16:09 +0300 Subject: [PATCH 127/220] Continue debugging --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index d164c7338..2c299ed6f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v' + -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From a7a9434398bbcc45b3a9eb8c6f13d64c8ba34f78 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:22:44 +0300 Subject: [PATCH 128/220] Remove env to test --- .github/workflows/release_build_and_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 2c299ed6f..a2258a0fa 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -38,9 +38,6 @@ jobs: docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ - -e GITHUB_ACTOR="${{ github.actor }}" \ - -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' From 7b91991a5ab7611d2aa38528ec9eecdd7ec7cd3e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:24:09 +0300 Subject: [PATCH 129/220] Removed safe dir --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a2258a0fa..0c8af74c9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -40,7 +40,7 @@ jobs: -w /workspace \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' + -c 'cat .git/config; set -x; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From ee9823822bafdd94146cde0c50762f0ddcb12f94 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:34:39 +0300 Subject: [PATCH 130/220] debug outpu --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 0c8af74c9..88ba0365f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -40,7 +40,7 @@ jobs: -w /workspace \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' + -c 'cat .git/config; set -x; whoami; ls -al;' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From 11fced5c5910a9d0ff292882430a3aaaf7d3c0cb Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:39:05 +0300 Subject: [PATCH 131/220] Debug output --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 88ba0365f..3412336b3 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -39,8 +39,9 @@ jobs: -v ${{ github.workspace }}:/workspace \ -w /workspace \ --entrypoint /bin/bash\ + --user $(id -u):0 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; whoami; ls -al;' + -c 'cat .git/config; set -x; id; whoami; ls -al;' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From abada2975fdb04d00ef9167d1004772a563b74f1 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:51:38 +0300 Subject: [PATCH 132/220] Add global safe dir and run tests --- .../workflows/build_release_automation.yml | 60 ++++++++++++++----- release-automation/docker/Dockerfile | 3 +- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index f20adf8eb..1aa495683 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -18,7 +18,7 @@ env: IMAGE_NAME: ${{ github.repository }}/release-automation jobs: - build-and-push: + build-test-and-push: runs-on: ubuntu-latest permissions: contents: read @@ -28,13 +28,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Log in to Container Registry - if: ${{ github.event.inputs.push_to_ghcr == 'true' }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Extract metadata id: meta @@ -46,24 +41,57 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=sha,prefix={{branch}}- - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push Docker image + - name: Build Docker image (without pushing) uses: docker/build-push-action@v5 with: context: ./release-automation - file: ./release-automation/docker/Dockerfile - push: ${{ github.event.inputs.push_to_ghcr == 'true' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + file: ./release-automation/docker/Dockerfile.prod + push: false + tags: test-image:latest + load: true cache-from: type=gha cache-to: type=gha,mode=max + - name: Test the built image + run: | + # Start container and install dev dependencies for testing + docker run --rm \ + -v ${{ github.workspace }}/release-automation:/workspace \ + -w /workspace \ + --entrypoint /bin/bash \ + test-image:latest \ + -c " + set -e + echo '=== Installing test dependencies ===' + pip install pytest pytest-cov + echo '=== Running tests ===' + pytest -v tests/ + " + + - name: Log in to Container Registry + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag and push image + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + run: | + # Tag the tested image with the proper tags + echo '${{ steps.meta.outputs.tags }}' | while read -r tag; do + docker tag test-image:latest "$tag" + docker push "$tag" + done + - name: Output image details run: | echo "## Docker Image Built Successfully! 🐳" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ… **Tests passed**" >> $GITHUB_STEP_SUMMARY + echo "šŸ—ļø **Production image built** (without dev dependencies)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY echo "**Tags:**" >> $GITHUB_STEP_SUMMARY echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/' | sed 's/$/`/' >> $GITHUB_STEP_SUMMARY diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile index 653200a2a..2d1c06650 100644 --- a/release-automation/docker/Dockerfile +++ b/release-automation/docker/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.11-slim-trixie RUN apt update && apt -y install git && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + && git config --global --add safe.directory '*' COPY . /release-automation RUN pip install -e /release-automation From e5cc7b32c0bc3c966d5d636392c9377f3aac53ee Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:54:13 +0300 Subject: [PATCH 133/220] Fix docker image --- .github/workflows/build_release_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 1aa495683..cb845ed8a 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -45,7 +45,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ./release-automation - file: ./release-automation/docker/Dockerfile.prod + file: ./release-automation/docker/Dockerfile push: false tags: test-image:latest load: true From 50884e52e4ee5eaffa1f10559ff56b249ff5a861 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:57:37 +0300 Subject: [PATCH 134/220] Try to fix workspace --- .github/workflows/build_release_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index cb845ed8a..6288b8b74 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -56,7 +56,7 @@ jobs: run: | # Start container and install dev dependencies for testing docker run --rm \ - -v ${{ github.workspace }}/release-automation:/workspace \ + -v ${{ github.workspace }}:/workspace \ -w /workspace \ --entrypoint /bin/bash \ test-image:latest \ From 35c8bc89aa71fa7023b908c144a71c9e8a802c9c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:59:01 +0300 Subject: [PATCH 135/220] cd to release-automation for testing --- .github/workflows/build_release_automation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 6288b8b74..3df169dd8 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -61,6 +61,7 @@ jobs: --entrypoint /bin/bash \ test-image:latest \ -c " + cd release-automation set -e echo '=== Installing test dependencies ===' pip install pytest pytest-cov From 7a430bd99d8a252a53768d19b13ab06fddf78317 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 20:34:38 +0300 Subject: [PATCH 136/220] Comment --- .github/workflows/build_release_automation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 3df169dd8..a9c0ff005 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -52,6 +52,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + # Integration tests do need access to git repository - name: Test the built image run: | # Start container and install dev dependencies for testing From f4f91bc6e108c264893cadd59edd837a81ebcc75 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 21:12:19 +0300 Subject: [PATCH 137/220] Full stackbrew format support --- .../src/stackbrew_generator/cli.py | 2 +- .../src/stackbrew_generator/distribution.py | 9 +++-- .../src/stackbrew_generator/models.py | 18 ++++++++- .../src/stackbrew_generator/stackbrew.py | 7 +++- .../src/stackbrew_generator/version_filter.py | 32 ++++++++-------- release-automation/tests/test_integration.py | 3 +- release-automation/tests/test_models.py | 32 ++++++++-------- release-automation/tests/test_stackbrew.py | 38 ++++++++++++------- 8 files changed, 86 insertions(+), 55 deletions(-) diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py index cdc1721b4..953a29048 100644 --- a/release-automation/src/stackbrew_generator/cli.py +++ b/release-automation/src/stackbrew_generator/cli.py @@ -81,7 +81,7 @@ def generate( raise typer.Exit(1) # Fetch required refs - refs_to_fetch = [commit for _, commit in versions] + refs_to_fetch = [commit for _, commit, _ in versions] git_client.fetch_refs(refs_to_fetch) # Prepare releases list with distribution information diff --git a/release-automation/src/stackbrew_generator/distribution.py b/release-automation/src/stackbrew_generator/distribution.py index 187e8f521..5e2080e8e 100644 --- a/release-automation/src/stackbrew_generator/distribution.py +++ b/release-automation/src/stackbrew_generator/distribution.py @@ -79,11 +79,11 @@ def get_distribution_for_commit(self, commit: str, distro_type: str) -> Distribu f"Failed to get distribution for {distro_type} from {commit}: {e}" ) from e - def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> List[Release]: + def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Release]: """Prepare list of releases with distribution information. Args: - versions: List of (RedisVersion, commit) tuples + versions: List of (RedisVersion, commit, tag_ref) tuples Returns: List of Release objects with distribution information @@ -93,7 +93,7 @@ def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> Lis releases = [] distro_types = ["debian", "alpine"] - for version, commit in versions: + for version, commit, tag_ref in versions: console.print(f"[dim]Processing [bold yellow]{version}[/bold yellow] - {commit[:8]}[/dim]") for distro_type in distro_types: @@ -103,7 +103,8 @@ def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> Lis release = Release( commit=commit, version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref=tag_ref ) releases.append(release) diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 599bbe9c9..37d2a1c89 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -154,6 +154,7 @@ class Release(BaseModel): commit: str = Field(..., description="Git commit hash") version: RedisVersion = Field(..., description="Redis version") distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") def __str__(self) -> str: """String representation of the release.""" @@ -171,7 +172,20 @@ class StackbrewEntry(BaseModel): commit: str = Field(..., description="Git commit hash") version: RedisVersion = Field(..., description="Redis version") distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") + + # Hard-coded architectures as specified + architectures: List[str] = Field( + default_factory=lambda: ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"], + description="Supported architectures" + ) def __str__(self) -> str: - """String representation as comma-separated tags.""" - return ", ".join(self.tags) + """String representation in stackbrew format.""" + lines = [] + lines.append(f"Tags: {', '.join(self.tags)}") + lines.append(f"Architectures: {', '.join(self.architectures)}") + lines.append(f"GitCommit: {self.commit}") + lines.append(f"GitFetch: {self.git_fetch_ref}") + lines.append(f"Directory: {self.distribution.type.value}") + return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index e40bd4a6a..5eb95213b 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -101,7 +101,8 @@ def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewE tags=tags, commit=release.commit, version=release.version, - distribution=release.distribution + distribution=release.distribution, + git_fetch_ref=release.git_fetch_ref ) entries.append(entry) @@ -125,7 +126,9 @@ def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: return "" lines = [] - for entry in entries: + for i, entry in enumerate(entries): + if i > 0: + lines.append("") # Add blank line between entries lines.append(str(entry)) return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index d928e0e83..23cb0ed3b 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -22,14 +22,14 @@ def __init__(self, git_client: GitClient): """ self.git_client = git_client - def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: """Get Redis versions from git tags. Args: major_version: Major version to filter for Returns: - List of (RedisVersion, commit) tuples sorted by version (newest first) + List of (RedisVersion, commit, tag_ref) tuples sorted by version (newest first) """ console.print(f"[blue]Getting Redis versions for major version {major_version}[/blue]") @@ -41,7 +41,7 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe for commit, tag_ref in tags: try: version = self.git_client.extract_version_from_tag(tag_ref, major_version) - versions.append((version, commit)) + versions.append((version, commit, tag_ref)) except Exception as e: console.print(f"[yellow]Warning: Skipping invalid tag {tag_ref}: {e}[/yellow]") continue @@ -52,11 +52,11 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") return versions - def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter out end-of-life versions. Args: - versions: List of (RedisVersion, commit) tuples + versions: List of (RedisVersion, commit, tag_ref) tuples Returns: Filtered list with EOL minor versions removed @@ -64,18 +64,18 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[ console.print("[blue]Filtering out EOL versions[/blue]") # Group versions by minor version - minor_versions: Dict[str, List[Tuple[RedisVersion, str]]] = {} - for version, commit in versions: + minor_versions: Dict[str, List[Tuple[RedisVersion, str, str]]] = {} + for version, commit, tag_ref in versions: minor_key = version.mainline_version if minor_key not in minor_versions: minor_versions[minor_key] = [] - minor_versions[minor_key].append((version, commit)) + minor_versions[minor_key].append((version, commit, tag_ref)) # Check each minor version for EOL marker filtered_versions = [] for minor_key, minor_group in minor_versions.items(): # Check if any version in this minor series is marked as EOL - has_eol = any(version.is_eol for version, _ in minor_group) + has_eol = any(version.is_eol for version, _, _ in minor_group) if has_eol: console.print(f"[yellow]Skipping minor version {minor_key}.* due to EOL[/yellow]") @@ -88,11 +88,11 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[ console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") return filtered_versions - def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter to keep only the latest patch version for each minor version and milestone status. Args: - versions: List of (RedisVersion, commit) tuples (should be sorted newest first) + versions: List of (RedisVersion, commit, tag_ref) tuples (should be sorted newest first) Returns: Filtered list with only the latest versions for each minor/milestone combination @@ -102,13 +102,13 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> Li seen_combinations = set() filtered_versions = [] - for version, commit in versions: + for version, commit, tag_ref in versions: # Create a key for minor version + milestone status combination_key = (version.mainline_version, version.is_milestone) if combination_key not in seen_combinations: seen_combinations.add(combination_key) - filtered_versions.append((version, commit)) + filtered_versions.append((version, commit, tag_ref)) milestone_str = "milestone" if version.is_milestone else "GA" console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") @@ -119,7 +119,7 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> Li console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") return filtered_versions - def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: """Get the actual Redis versions to process for a major version. This is the main entry point that combines all filtering steps: @@ -131,7 +131,7 @@ def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[Redi major_version: Major version to process Returns: - List of (RedisVersion, commit) tuples for processing + List of (RedisVersion, commit, tag_ref) tuples for processing """ console.print(f"[bold blue]Processing Redis {major_version}.x versions[/bold blue]") @@ -147,7 +147,7 @@ def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[Redi versions = self.filter_actual_versions(versions) console.print(f"[green]Final selection: {len(versions)} versions to process[/green]") - for version, commit in versions: + for version, commit, tag_ref in versions: console.print(f"[green] [bold yellow]{version}[/bold yellow] - {commit[:8]}[/green]") return versions diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py index 066392eaf..4d5ebd2a4 100644 --- a/release-automation/tests/test_integration.py +++ b/release-automation/tests/test_integration.py @@ -44,7 +44,8 @@ def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distributio Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ) ] mock_distribution_detector.prepare_releases_list.return_value = mock_releases diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py index bc55fa9ab..1ca03a1ab 100644 --- a/release-automation/tests/test_models.py +++ b/release-automation/tests/test_models.py @@ -53,7 +53,7 @@ def test_parse_invalid_version(self): """Test parsing invalid version strings.""" with pytest.raises(ValueError): RedisVersion.parse("invalid") - + with pytest.raises(ValueError): RedisVersion.parse("0.1.0") # Major version must be >= 1 @@ -61,7 +61,7 @@ def test_is_milestone(self): """Test milestone detection.""" ga_version = RedisVersion.parse("8.2.1") milestone_version = RedisVersion.parse("8.2.1-m01") - + assert ga_version.is_milestone is False assert milestone_version.is_milestone is True @@ -75,7 +75,7 @@ def test_string_representation(self): version1 = RedisVersion.parse("8.2.1") version2 = RedisVersion.parse("8.2.1-m01") version3 = RedisVersion.parse("8.2") - + assert str(version1) == "8.2.1" assert str(version2) == "8.2.1-m01" assert str(version3) == "8.2" @@ -86,14 +86,14 @@ def test_version_comparison(self): v2 = RedisVersion.parse("8.2.2") v3 = RedisVersion.parse("8.2.1-m01") v4 = RedisVersion.parse("8.3.0") - + # Test numeric comparison assert v1 < v2 assert v2 < v4 - + # Test milestone vs GA (GA comes after milestone) assert v3 < v1 - + # Test sorting versions = [v4, v1, v3, v2] sorted_versions = sorted(versions) @@ -125,7 +125,7 @@ def test_from_dockerfile_invalid(self): """Test parsing invalid Dockerfile lines.""" with pytest.raises(ValueError): Distribution.from_dockerfile_line("INVALID LINE") - + with pytest.raises(ValueError): Distribution.from_dockerfile_line("FROM unsupported:latest") @@ -133,7 +133,7 @@ def test_is_default(self): """Test default distribution detection.""" alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") debian = Distribution(type=DistroType.DEBIAN, name="bookworm") - + assert alpine.is_default is False assert debian.is_default is True @@ -141,7 +141,7 @@ def test_tag_names(self): """Test tag name generation.""" alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") debian = Distribution(type=DistroType.DEBIAN, name="bookworm") - + assert alpine.tag_names == ["alpine", "alpine3.22"] assert debian.tag_names == ["bookworm"] @@ -153,13 +153,14 @@ def test_release_creation(self): """Test creating a Release instance.""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - + release = Release( commit="abc123def456", version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" ) - + assert release.commit == "abc123def456" assert release.version == version assert release.distribution == distribution @@ -168,12 +169,13 @@ def test_release_string_representation(self): """Test Release string representation.""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - + release = Release( commit="abc123def456", version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" ) - + expected = "abc123de 8.2.1 debian bookworm" assert str(release) == expected diff --git a/release-automation/tests/test_stackbrew.py b/release-automation/tests/test_stackbrew.py index 2d065513e..e0864a41f 100644 --- a/release-automation/tests/test_stackbrew.py +++ b/release-automation/tests/test_stackbrew.py @@ -15,7 +15,7 @@ def test_generate_tags_debian_ga_latest(self): """Test tag generation for Debian GA version (latest).""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") tags = self.generator.generate_tags_for_release(release, is_latest=True) @@ -36,7 +36,7 @@ def test_generate_tags_debian_ga_not_latest(self): """Test tag generation for Debian GA version (not latest).""" version = RedisVersion.parse("7.4.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v7.4.1") tags = self.generator.generate_tags_for_release(release, is_latest=False) @@ -53,7 +53,7 @@ def test_generate_tags_alpine_ga_latest(self): """Test tag generation for Alpine GA version (latest).""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") tags = self.generator.generate_tags_for_release(release, is_latest=True) @@ -74,7 +74,7 @@ def test_generate_tags_milestone_version(self): """Test tag generation for milestone version.""" version = RedisVersion.parse("8.2.1-m01") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1-m01") tags = self.generator.generate_tags_for_release(release, is_latest=False) @@ -94,17 +94,20 @@ def test_generate_stackbrew_library(self): Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22") + distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="def456", version=RedisVersion.parse("8.1.5"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.1.5" ) ] @@ -128,7 +131,8 @@ def test_format_stackbrew_output(self): Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ) ] @@ -147,32 +151,38 @@ def test_generate_stackbrew_library_with_head_milestone(self): Release( commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", version=RedisVersion.parse("8.2.2-m01-int1"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" ), Release( commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", version=RedisVersion.parse("8.2.2-m01-int1"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" ), Release( commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", version=RedisVersion.parse("8.2.1"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", version=RedisVersion.parse("8.2.1"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="101262a8cf05b98137d88bc17e77db90c24cc783", version=RedisVersion.parse("8.0.3"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.0.3" ), Release( commit="101262a8cf05b98137d88bc17e77db90c24cc783", version=RedisVersion.parse("8.0.3"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.21") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.21"), + git_fetch_ref="refs/tags/v8.0.3" ) ] From c140a5bee7283a5b41079b0c8bafe9856802ed0a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 09:57:19 +0300 Subject: [PATCH 138/220] Try interactive mode --- .github/workflows/release_build_and_test.yml | 13 +------------ release-automation/docker/Dockerfile | 1 + 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3412336b3..c982d9449 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,20 +35,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm \ + docker run --rm -it \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - --entrypoint /bin/bash\ - --user $(id -u):0 \ - $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; id; whoami; ls -al;' - - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - -w /workspace \ - -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ - -e GITHUB_ACTOR="${{ github.actor }}" \ - -e GITHUB_REPOSITORY="${{ github.repository }}" \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile index 2d1c06650..2a6f061c2 100644 --- a/release-automation/docker/Dockerfile +++ b/release-automation/docker/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.11-slim-trixie RUN apt update && apt -y install git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + # avoid dubious permissions problem in github CI && git config --global --add safe.directory '*' COPY . /release-automation From 45aa7b15d32a568e984c0aa220a833f12672486d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 10:02:00 +0300 Subject: [PATCH 139/220] Use only -i --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c982d9449..444b76788 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,7 +35,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm -it \ + docker run --rm -i \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ From 688c2c33a135ef80d34ddc54a2ccaef7ead14415 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 10:05:45 +0300 Subject: [PATCH 140/220] Try force color --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 444b76788..e8711056c 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,9 +35,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm -i \ + docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ + -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 From 1b9bf327e42c0de082db04adc78080f052f0caa4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 14:10:11 +0300 Subject: [PATCH 141/220] Readme --- release-automation/README.md | 59 +++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/release-automation/README.md b/release-automation/README.md index 70aec8fce..e37e7189a 100644 --- a/release-automation/README.md +++ b/release-automation/README.md @@ -1,4 +1,55 @@ -# Release automation helper +# Docker release process and release automation description + +This readme covers relase process for versions 8 and above. + +In version 8 the docker-library structure has changed. Static Dockerfiles are used instead of templating. Versions live in a different mainline branches and are marked with tags. + +The docker release process goal is to create a PR in official-docker library for library/redis file. + +library/redis stackbrew file should reflect the tags in redis/docker-library-redis repository. + +## Branches and tags + +Mainline branches are named `release/Major.Minor` (e.g. `release/8.2`) + +Each version release is tagged with `vMajor.Minor.Patch` (e.g. `v8.2.1`) + +Milestone releases are tagged with `vMajor.Minor.Patch-Milestone` (e.g. `v8.2.1-m01`). Any suffix after patch version is considered a milestone. + +Tags without suffix are considered GA (General Availability) releases (e.g. `v8.2.1`). + +Internal releases are milestone releases containing `-int` in their name (e.g. `v8.2.1-m01-int1` or `8.4.0-int3`). They are not released to the public. + +Milestone releases never get latest or any other default tags, like `8`, `8.2`, `8.2.1`, `latest`, `bookworm`, etc. + +For each mainline only one GA release and optionally one milestone release should be published in official-library. The most latest versions. + +End of life versions are marked with `-eol` suffix (e.g. `v8.0.3-eol`). When there is a at least one minor version tagged with eol all versions in this minor series are considered EOL and are not included in the release file. + +## Creating a release manually + +This process is automated using github workflows. However, it's useful to understand the manual process. + +Determine a mainline branch, e.g `release/8.2` for version `8.2.2`. + +Optionally create a release branch from the mainline branch, e.g. `8.2.2`. + +Modify dockerfiles. + +Test dockerfiles. + +If release branch was created, merge it back to mainline branch. + +Tag commit with `vMajor.Minor.Patch` (e.g. `v8.2.1`) in the mainline branch. + +Push your changes to redis/docker-library-redis repository. + +Create a PR to official-library refering the tag and commit you created. + + +# Release automation tool + +Release automation tool is used to generate library/redis file for official-library. It uses origin repository as a source of truth and follows the process described above. ## Installation @@ -15,3 +66,9 @@ pip install -e . cd release-automation pip install -e ".[dev]" ``` + +## Usage + +```bash +release-automation --help +``` From 77a5c189520dc5a4e7b680bf412dadeace0a35f4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 15:37:19 +0300 Subject: [PATCH 142/220] Refactoring the code --- .../src/stackbrew_generator/cli.py | 207 +++++++++++++----- .../src/stackbrew_generator/models.py | 15 +- .../src/stackbrew_generator/stackbrew.py | 160 ++++++++++++++ release-automation/tests/test_integration.py | 52 +---- release-automation/tests/test_models.py | 60 ++++- 5 files changed, 391 insertions(+), 103 deletions(-) diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py index 953a29048..f572bee7a 100644 --- a/release-automation/src/stackbrew_generator/cli.py +++ b/release-automation/src/stackbrew_generator/cli.py @@ -1,6 +1,7 @@ """CLI interface for stackbrew library generator.""" import typer +from pathlib import Path from rich.console import Console from rich.traceback import install @@ -8,7 +9,7 @@ from .exceptions import StackbrewGeneratorError from .git_operations import GitClient from .logging_config import setup_logging -from .stackbrew import StackbrewGenerator +from .stackbrew import StackbrewGenerator, StackbrewUpdater from .version_filter import VersionFilter # Install rich traceback handler @@ -24,8 +25,63 @@ console = Console(stderr=True) -@app.command() -def generate( +def _generate_stackbrew_content(major_version: int, remote: str, verbose: bool) -> str: + """Generate stackbrew content for a major version. + + This helper function contains the common logic for generating stackbrew content + that is used by both generate-stackbrew-content and update-stackbrew-file commands. + + Args: + major_version: Redis major version to process + remote: Git remote to use + verbose: Whether to enable verbose output + + Returns: + Generated stackbrew content as string + + Raises: + typer.Exit: If no versions found or other errors occur + """ + # Initialize components + git_client = GitClient(remote=remote) + version_filter = VersionFilter(git_client) + distribution_detector = DistributionDetector(git_client) + stackbrew_generator = StackbrewGenerator() + + # Get actual Redis versions to process + versions = version_filter.get_actual_major_redis_versions(major_version) + + if not versions: + console.print(f"[red]No versions found for Redis {major_version}.x[/red]") + raise typer.Exit(1) + + # Fetch required refs + refs_to_fetch = [commit for _, commit, _ in versions] + git_client.fetch_refs(refs_to_fetch) + + # Prepare releases list with distribution information + releases = distribution_detector.prepare_releases_list(versions) + + if not releases: + console.print("[red]No releases prepared[/red]") + raise typer.Exit(1) + + # Generate stackbrew library content + entries = stackbrew_generator.generate_stackbrew_library(releases) + output = stackbrew_generator.format_stackbrew_output(entries) + + if not output: + console.print("[yellow]No stackbrew content generated[/yellow]") + raise typer.Exit(1) + + if verbose: + console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + + return output + + +@app.command(name="generate-stackbrew-content") +def generate_stackbrew_content( major_version: int = typer.Argument( ..., help="Redis major version to process (e.g., 8 for Redis 8.x)" @@ -41,11 +97,6 @@ def generate( "-v", help="Enable verbose output" ), - dry_run: bool = typer.Option( - False, - "--dry-run", - help="Show generated content without outputting to stdout" - ), ) -> None: """Generate stackbrew library content for Redis Docker images. @@ -63,52 +114,117 @@ def generate( console.print(f"[bold blue]Stackbrew Library Generator[/bold blue]") console.print(f"Major version: {major_version}") console.print(f"Remote: {remote}") - if dry_run: - console.print("[yellow]DRY RUN MODE - Generated content will be shown but not output to stdout[/yellow]") try: - # Initialize components - git_client = GitClient(remote=remote) - version_filter = VersionFilter(git_client) - distribution_detector = DistributionDetector(git_client) - stackbrew_generator = StackbrewGenerator() + # Generate stackbrew content using the helper function + output = _generate_stackbrew_content(major_version, remote, verbose) - # Get actual Redis versions to process - versions = version_filter.get_actual_major_redis_versions(major_version) + # Output the stackbrew library content + print(output) - if not versions: - console.print(f"[red]No versions found for Redis {major_version}.x[/red]") - raise typer.Exit(1) + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) - # Fetch required refs - refs_to_fetch = [commit for _, commit, _ in versions] - git_client.fetch_refs(refs_to_fetch) - # Prepare releases list with distribution information - releases = distribution_detector.prepare_releases_list(versions) +@app.command() +def version() -> None: + """Show version information.""" + from . import __version__ + console.print(f"stackbrew-library-generator {__version__}") - if not releases: - console.print("[red]No releases prepared[/red]") - raise typer.Exit(1) - # Generate stackbrew library content - entries = stackbrew_generator.generate_stackbrew_library(releases) - output = stackbrew_generator.format_stackbrew_output(entries) +@app.command() +def update_stackbrew_file( + major_version: int = typer.Argument( + ..., + help="Redis major version to update (e.g., 8 for Redis 8.x)" + ), + input_file: Path = typer.Option( + ..., + "--input", + "-i", + help="Path to the stackbrew library file to update" + ), + output_file: Path = typer.Option( + None, + "--output", + "-o", + help="Output file path (defaults to stdout)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), +) -> None: + """Update stackbrew library file by replacing entries for a specific major version. - if dry_run: - console.print(f"[yellow]DRY RUN: Would generate stackbrew library with {len(entries)} entries[/yellow]") - if verbose: - console.print("[yellow]Generated content:[/yellow]") - console.print(output) + This command: + 1. Reads the existing stackbrew library file + 2. Generates new stackbrew content for the specified major version + 3. Replaces all entries related to that major version in their original position + 4. Preserves the header and entries for other major versions + 5. Outputs to stdout by default, or to specified output file + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if not input_file.exists(): + console.print(f"[red]Input file does not exist: {input_file}[/red]") + raise typer.Exit(1) + + if verbose: + console.print(f"[bold blue]Stackbrew Library File Updater[/bold blue]") + console.print(f"Input file: {input_file}") + if output_file: + console.print(f"Output file: {output_file}") else: - if output: - # Output the stackbrew library content - print(output) + console.print("Output: stdout") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") - if verbose: - console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + try: + # Generate new stackbrew content for the major version using helper function + new_content = _generate_stackbrew_content(major_version, remote, verbose) + + # Update the stackbrew file content + updater = StackbrewUpdater() + updated_content = updater.update_stackbrew_content( + input_file, major_version, new_content, verbose + ) + + # Write the updated content + if output_file: + output_file.write_text(updated_content, encoding='utf-8') + if verbose: + console.print(f"[green]Successfully updated {output_file} for Redis {major_version}.x[/green]") else: - console.print("[yellow]No stackbrew content generated[/yellow]") + console.print(f"[green]Updated {output_file}[/green]") + else: + # Output to stdout + print(updated_content) + if verbose: + console.print(f"[green]Generated updated stackbrew content for Redis {major_version}.x[/green]") except StackbrewGeneratorError as e: if verbose and hasattr(e, 'get_detailed_message'): @@ -128,12 +244,5 @@ def generate( raise typer.Exit(1) -@app.command() -def version() -> None: - """Show version information.""" - from . import __version__ - console.print(f"stackbrew-library-generator {__version__}") - - if __name__ == "__main__": app() diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 37d2a1c89..6d7c88991 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -174,11 +174,16 @@ class StackbrewEntry(BaseModel): distribution: Distribution = Field(..., description="Linux distribution") git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") - # Hard-coded architectures as specified - architectures: List[str] = Field( - default_factory=lambda: ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"], - description="Supported architectures" - ) + @property + def architectures(self) -> List[str]: + """Get supported architectures based on distribution type.""" + if self.distribution.type == DistroType.DEBIAN: + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + elif self.distribution.type == DistroType.ALPINE: + return ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + else: + # Fallback to debian architectures for unknown distributions + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] def __str__(self) -> str: """String representation in stackbrew format.""" diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index 5eb95213b..d872b65e5 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -1,5 +1,7 @@ """Stackbrew library generation.""" +import re +from pathlib import Path from typing import List from rich.console import Console @@ -132,3 +134,161 @@ def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: lines.append(str(entry)) return "\n".join(lines) + + +class StackbrewUpdater: + """Updates stackbrew library files by replacing entries for specific major versions.""" + + def __init__(self): + """Initialize the updater.""" + pass + + def update_stackbrew_content(self, input_file: Path, major_version: int, new_content: str, verbose: bool = False) -> str: + """Update stackbrew file content by replacing entries for a specific major version. + + Args: + input_file: Path to the input stackbrew file + major_version: Major version to replace entries for + new_content: New stackbrew content to insert + verbose: Whether to print verbose output + + Returns: + Updated stackbrew file content + """ + content = input_file.read_text(encoding='utf-8') + lines = content.split('\n') + + # Find header (everything before the first Tags: line) + header_lines = [] + content_start_idx = 0 + + for i, line in enumerate(lines): + if line.startswith('Tags:'): + content_start_idx = i + break + header_lines.append(line) + + if content_start_idx == 0 and not any(line.startswith('Tags:') for line in lines): + # No existing entries, just append new content + if verbose: + console.print("[dim]No existing entries found, appending new content[/dim]") + return content.rstrip() + '\n\n' + new_content + + # Parse entries and find where target major version entries start and end + entries = self._parse_stackbrew_entries(lines[content_start_idx:]) + target_entries = [] + other_entries_before = [] + other_entries_after = [] + target_start_found = False + target_end_found = False + removed_count = 0 + + for entry in entries: + if self._entry_belongs_to_major_version(entry, major_version): + target_entries.append(entry) + removed_count += 1 + if not target_start_found: + target_start_found = True + elif not target_start_found: + # Entries before target major version + other_entries_before.append(entry) + else: + # Entries after target major version + other_entries_after.append(entry) + if not target_end_found: + target_end_found = True + + if verbose: + if removed_count > 0: + console.print(f"[dim]Removed {removed_count} existing entries for Redis {major_version}.x[/dim]") + else: + console.print(f"[dim]No existing entries found for Redis {major_version}.x, placing at end[/dim]") + + # Reconstruct the file + result_lines = header_lines[:] + + # Add entries before target major version + for entry in other_entries_before: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + # Add new content for the target major version + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(new_content.split('\n')) + + # Add entries after target major version + for entry in other_entries_after: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + return '\n'.join(result_lines) + + def _parse_stackbrew_entries(self, lines: List[str]) -> List[List[str]]: + """Parse stackbrew entries from lines, returning list of entry line groups. + + Args: + lines: Lines to parse + + Returns: + List of entry line groups + """ + entries = [] + current_entry = [] + + for line in lines: + line = line.rstrip() + + if line.startswith('Tags:') and current_entry: + # Start of new entry, save the previous one + entries.append(current_entry) + current_entry = [line] + elif line.startswith('Tags:'): + # First entry + current_entry = [line] + elif current_entry and (line.startswith(('Architectures:', 'GitCommit:', 'GitFetch:', 'Directory:')) or line.strip() == ''): + # Part of current entry + current_entry.append(line) + elif not line.strip() and not current_entry: + # Empty line before any entry starts, skip + continue + elif not line.strip() and current_entry: + # Empty line after entry content - end of entry + if current_entry: + entries.append(current_entry) + current_entry = [] + + # Don't forget the last entry + if current_entry: + entries.append(current_entry) + + return entries + + def _entry_belongs_to_major_version(self, entry_lines: List[str], major_version: int) -> bool: + """Check if a stackbrew entry belongs to the specified major version. + + Args: + entry_lines: Lines of the stackbrew entry + major_version: Major version to check for + + Returns: + True if the entry belongs to the major version + """ + for line in entry_lines: + if line.startswith('Tags:'): + tags_line = line[5:].strip() # Remove 'Tags:' prefix + tags = [tag.strip() for tag in tags_line.split(',')] + + # Check if any tag indicates this major version + for tag in tags: + # Look for patterns like "8", "8.2", "8.2.1", "8-alpine", etc. + if re.match(rf'^{major_version}(?:\.|$|-)', tag): + return True + # Also check for "latest" tag which typically belongs to the highest major version + # But we'll be conservative and not assume latest belongs to our major version + # unless we have other evidence + break + + return False diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py index 4d5ebd2a4..2a11548a1 100644 --- a/release-automation/tests/test_integration.py +++ b/release-automation/tests/test_integration.py @@ -15,49 +15,6 @@ def setup_method(self): """Set up test fixtures.""" self.runner = CliRunner() - @patch('stackbrew_generator.distribution.DistributionDetector') - @patch('stackbrew_generator.git_operations.GitClient') - def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distribution_detector_class): - """Test complete workflow in dry run mode.""" - # Mock git client - mock_git_client = Mock() - mock_git_client_class.return_value = mock_git_client - - # Mock distribution detector - mock_distribution_detector = Mock() - mock_distribution_detector_class.return_value = mock_distribution_detector - - # Mock git operations - mock_git_client.list_remote_tags.return_value = [ - ("abc123", "refs/tags/v8.2.1"), - ("def456", "refs/tags/v8.2.0"), - ] - - mock_git_client.extract_version_from_tag.side_effect = [ - RedisVersion.parse("8.2.1"), - RedisVersion.parse("8.2.0"), - ] - - # Mock releases - from stackbrew_generator.models import Release, Distribution, DistroType - mock_releases = [ - Release( - commit="abc123", - version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), - git_fetch_ref="refs/tags/v8.2.1" - ) - ] - mock_distribution_detector.prepare_releases_list.return_value = mock_releases - - # Run command in dry run mode - result = self.runner.invoke(app, ["generate", "8", "--dry-run", "--verbose"]) - - # Check that it completed successfully - assert result.exit_code == 0 - assert "DRY RUN: Would generate stackbrew library" in result.stderr - assert "Generated content:" in result.stderr - def test_version_command(self): """Test version command.""" result = self.runner.invoke(app, ["version"]) @@ -66,7 +23,7 @@ def test_version_command(self): def test_invalid_major_version(self): """Test handling of invalid major version.""" - result = self.runner.invoke(app, ["generate", "0"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "0"]) assert result.exit_code != 0 @patch('stackbrew_generator.git_operations.GitClient') @@ -77,7 +34,7 @@ def test_no_tags_found(self, mock_git_client_class): mock_git_client_class.return_value = mock_git_client mock_git_client.list_remote_tags.return_value = [] - result = self.runner.invoke(app, ["generate", "99"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "99"]) assert result.exit_code == 1 assert "No tags found" in result.stderr @@ -87,15 +44,14 @@ def test_no_versions_found(self, mock_get_versions): # Mock git client to return no tags mock_get_versions.return_value = [] - result = self.runner.invoke(app, ["generate", "8"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "8"]) #assert result.exit_code == 1 assert "No versions found" in result.stderr def test_help_output(self): """Test help output.""" - result = self.runner.invoke(app, ["generate", "--help"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "--help"]) assert result.exit_code == 0 assert "Generate stackbrew library content" in result.stdout assert "--remote" in result.stdout assert "--verbose" in result.stdout - assert "--dry-run" in result.stdout diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py index 1ca03a1ab..0feef1f94 100644 --- a/release-automation/tests/test_models.py +++ b/release-automation/tests/test_models.py @@ -2,7 +2,7 @@ import pytest -from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release, StackbrewEntry class TestRedisVersion: @@ -179,3 +179,61 @@ def test_release_string_representation(self): expected = "abc123de 8.2.1 debian bookworm" assert str(release) == expected + + +class TestStackbrewEntry: + """Tests for StackbrewEntry model.""" + + def test_debian_architectures(self): + """Test that Debian distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + entry = StackbrewEntry( + tags=["8.2.1", "latest"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + assert entry.architectures == expected_architectures + + def test_alpine_architectures(self): + """Test that Alpine distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + assert entry.architectures == expected_architectures + + def test_stackbrew_entry_string_format(self): + """Test that StackbrewEntry formats correctly with architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + output = str(entry) + + # Check that it contains the expected Alpine architectures + assert "amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x" in output + assert "Tags: 8.2.1-alpine, alpine" in output + assert "GitCommit: abc123def456" in output + assert "GitFetch: refs/tags/v8.2.1" in output + assert "Directory: alpine" in output From 316f0c4783b6ca89a0e225d11bcd1566424a06bf Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 17:01:20 +0300 Subject: [PATCH 143/220] Temporary disable the build, test tag creation --- .github/workflows/release_build_and_test.yml | 46 ++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index e8711056c..0013bc9b0 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -31,17 +31,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Test Release Automation Docker Image - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - -w /workspace \ - -e FORCE_COLOR=1 \ - $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - generate 8 - - name: Validate Redis Release Archive uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: @@ -63,12 +52,16 @@ jobs: release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: + runs-on: ["ubuntu-latest"] + steps: + - shell: bash + run: | + echo Hi needs: prepare-release - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} - + #uses: ./.github/workflows/pre-merge.yml + #secrets: inherit + #with: + # release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] @@ -87,8 +80,27 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch + id: merge-back uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} - gh_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create version tag + uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + with: + tag: v${{ github.event.inputs.release_tag }} + ref: ${{ steps.merge-back.outputs.merge_commit_sha }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Test Release Automation Docker Image + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # run: | + # docker run --rm \ + # -v ${{ github.workspace }}:/workspace \ + # -w /workspace \ + # -e FORCE_COLOR=1 \ + # $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ + # generate 8 \ No newline at end of file From 4c0e47005d3a64360e660c53f7471b6374c8f8dd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 17:42:59 +0300 Subject: [PATCH 144/220] Create tag only if merge was done --- .github/workflows/release_build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 0013bc9b0..c029ae4f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -88,6 +88,7 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Create version tag + if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main with: tag: v${{ github.event.inputs.release_tag }} From 9faae5f7b6cafedaa99f51f899f36f10d6b4cf58 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 22:23:47 +0300 Subject: [PATCH 145/220] Debug output --- .github/workflows/release_build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c029ae4f6..caf9e94b9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,6 +87,14 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} + - name: Debug + shell: bash + run: | + echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" + echo "Merge commit SHA length: ${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c + echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : + echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} + - name: Create version tag if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main From 0631debc12aec9f62e7a9c3edfecd2e20b39c147 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:02:38 +0300 Subject: [PATCH 146/220] lilbit dbg --- .github/workflows/release_build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index caf9e94b9..408471261 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -91,8 +91,8 @@ jobs: shell: bash run: | echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: ${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c - echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : + echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}"" | wc -c) + echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} - name: Create version tag From c81711689d3d2ba04ea467ad62aeb01303e7f4ad Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:04:46 +0300 Subject: [PATCH 147/220] fix dblquoute --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 408471261..5c33be479 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -91,7 +91,7 @@ jobs: shell: bash run: | echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}"" | wc -c) + echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c) echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} From 9d6435dca10190e7ec70ca9705e777eca3e472be Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:09:51 +0300 Subject: [PATCH 148/220] Remove debug --- .github/workflows/release_build_and_test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5c33be479..c029ae4f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,14 +87,6 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - name: Debug - shell: bash - run: | - echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c) - echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : - echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} - - name: Create version tag if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main From c626c11407daf99c9bb64ce97de6424e163d4f58 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 12:33:07 +0300 Subject: [PATCH 149/220] Release handle --- .github/workflows/release_build_and_test.yml | 43 +++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c029ae4f6..db0500579 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,13 +87,44 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - name: Create version tag - if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} - uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + - name: Create release handle JSON + shell: bash + run: | + if [ -n "${{ steps.merge-back.outputs.merge_commit_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.merge_commit_sha }}" + elif [ -n "${{ steps.merge-back.outputs.target_before_merge_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.target_before_merge_sha }}" + else + echo "Error: No commit SHA found, both merge_commit_sha and target_before_merge_sha are empty" >&2 + exit 1 + fi + + cat > release_handle.json << EOF + { + "release_commit_sha": "$RELEASE_COMMIT_SHA", + "release_version": "${{ github.event.inputs.release_tag }}", + "release_version_branch": "${{ steps.ensure-branch.outputs.release_version_branch }}", + "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}" + } + EOF + + echo "Created release_handle.json:" + cat release_handle.json + + - name: Upload release handle artifact + uses: actions/upload-artifact@v4 with: - tag: v${{ github.event.inputs.release_tag }} - ref: ${{ steps.merge-back.outputs.merge_commit_sha }} - gh_token: ${{ secrets.GITHUB_TOKEN }} + name: release-handle + path: release_handle.json + retention-days: 30 + + # - name: Create version tag + # if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} + # uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + # with: + # tag: v${{ github.event.inputs.release_tag }} + # ref: ${{ steps.merge-back.outputs.merge_commit_sha }} + # gh_token: ${{ secrets.GITHUB_TOKEN }} # - name: Test Release Automation Docker Image # env: From 256448ccae3d1e698cc27aa001ddc3111e0e47f6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 14:01:23 +0300 Subject: [PATCH 150/220] Rename release_handle --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index db0500579..75d8e4ccd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -114,7 +114,7 @@ jobs: - name: Upload release handle artifact uses: actions/upload-artifact@v4 with: - name: release-handle + name: release_handle path: release_handle.json retention-days: 30 From 8ccb38d912c93ac4451131a03f98a113c7ed1f12 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:16:17 +0300 Subject: [PATCH 151/220] Release publish first run --- .github/workflows/release_publish.yml | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/release_publish.yml diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml new file mode 100644 index 000000000..4b902309e --- /dev/null +++ b/.github/workflows/release_publish.yml @@ -0,0 +1,69 @@ +# This workflow publishes a release by creating a version tag. +# It is intended to be run with workflow_dispatch event by the automation. + +on: + workflow_dispatch: + inputs: + release_handle: + description: 'Release handle JSON string containing release information' + required: true + type: string + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false + push: + branches: + - release/8* + +# UUID is used to help automation to identify workflow run in the list of workflow runs. +run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" + +jobs: + publish-release: + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Parse release handle and validate + id: parse-release + shell: bash + run: | + # Parse the JSON input + RELEASE_HANDLE='${{ github.event.inputs.release_handle }}' + echo "Parsing release handle JSON:" + echo "$RELEASE_HANDLE" | jq . + + # Extract release_commit_sha + RELEASE_COMMIT_SHA=$(echo "$RELEASE_HANDLE" | jq -r '.release_commit_sha // empty') + + # Validate that release_commit_sha exists and is not empty + if [ -z "$RELEASE_COMMIT_SHA" ] || [ "$RELEASE_COMMIT_SHA" = "null" ]; then + echo "Error: release_commit_sha is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + # Extract release_version for tag creation + RELEASE_VERSION=$(echo "$RELEASE_HANDLE" | jq -r '.release_version // empty') + + if [ -z "$RELEASE_VERSION" ] || [ "$RELEASE_VERSION" = "null" ]; then + echo "Error: release_version is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + echo "Successfully parsed release handle:" + echo " release_commit_sha: $RELEASE_COMMIT_SHA" + echo " release_version: $RELEASE_VERSION" + + # Set outputs for next steps + echo "release_commit_sha=$RELEASE_COMMIT_SHA" >> $GITHUB_OUTPUT + echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT + + - name: Create version tag + uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + with: + tag: v${{ steps.parse-release.outputs.release_version }} + ref: ${{ steps.parse-release.outputs.release_commit_sha }} + gh_token: ${{ secrets.GITHUB_TOKEN }} From 9ab8c8eee58b59c05b7cf40ed0ec75b4b6738ecb Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:18:07 +0300 Subject: [PATCH 152/220] Fix push condition --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 4b902309e..70dd798a1 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,7 +13,7 @@ on: required: false push: branches: - - release/8* + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 2613793b942b2b8c0c681b40bacbfb68e1aedbc9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:21:38 +0300 Subject: [PATCH 153/220] Fix yaml for release_publish workflow --- .github/workflows/release_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 70dd798a1..dc4d91047 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -11,9 +11,9 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** + push: + branches: + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 3e6e240fd56fd958656794a87458bccd3c8b67e6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:22:40 +0300 Subject: [PATCH 154/220] Remove push run --- .github/workflows/release_publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index dc4d91047..d0f0f111c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -11,9 +11,6 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From eb29459d4a31b7656d9c89b640c9a17a2cd46522 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 20:54:23 +0300 Subject: [PATCH 155/220] Creating a PR: first attempt --- .github/workflows/release_publish.yml | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index d0f0f111c..cc7f03c98 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -64,3 +64,47 @@ jobs: tag: v${{ steps.parse-release.outputs.release_version }} ref: ${{ steps.parse-release.outputs.release_commit_sha }} gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout official-images repo + uses: actions/checkout@v4 + with: + path: official-images + repository: Sh-Peter/official-library + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate stackbrew library content + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Extract major version from release version (e.g., "8.2.1" -> "8") + MAJOR_VERSION=$(echo "${{ steps.parse-release.outputs.release_version }}" | cut -d. -f1) + echo "Major version: $MAJOR_VERSION" + + # Generate updated stackbrew content using the release automation Docker image + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e FORCE_COLOR=1 \ + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ + update-stackbrew-file --input official-images/library/redis --output official-images/library/redis + + - name: Create pull request to official-images + id: create-pr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + push-to-fork: Peter-Sh/official-images + path: official-images + branch: redis-${{ steps.parse-release.outputs.release_version }} + commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + body: | + Automated update for Redis ${{ steps.parse-release.outputs.release_version }} + + Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} + Release tag: v${{ steps.parse-release.outputs.release_version }} + + - name: PR creation results + run: | + echo "Pull Request Number: ${{ steps.create-pr.outputs.pull-request-number }}" + echo "Pull Request URL: ${{ steps.create-pr.outputs.pull-request-url }}" From 2291feeb2dc3d4d1c48a0c1ae677a75b97b9b6b6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:11:10 +0300 Subject: [PATCH 156/220] Fix official images url --- .github/workflows/release_publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index cc7f03c98..82e20de30 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -69,8 +69,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Sh-Peter/official-library - token: ${{ secrets.GITHUB_TOKEN }} + repository: Sh-Peter/official-images - name: Generate stackbrew library content env: From fd6c12f317f34d9be4388251a94382c7aa650259 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:18:56 +0300 Subject: [PATCH 157/220] Fix update-stackbrew-file args --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 82e20de30..3cabb1f11 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -85,7 +85,7 @@ jobs: -w /workspace \ -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ - update-stackbrew-file --input official-images/library/redis --output official-images/library/redis + update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis - name: Create pull request to official-images id: create-pr From 75794726da9a23c51977aa7abb5110527f5c8704 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:36:10 +0300 Subject: [PATCH 158/220] Swap remotes --- .github/workflows/release_publish.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 3cabb1f11..b6a4dcdb4 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Sh-Peter/official-images + repository: Peter-Sh/official-images - name: Generate stackbrew library content env: @@ -86,13 +86,15 @@ jobs: -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis + cd official-images && git diff + cd - - name: Create pull request to official-images id: create-pr uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} - push-to-fork: Peter-Sh/official-images + push-to-fork: Sh-Peter/official-images path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" From 480669b1fb60b421cd0981b7f515107f7ed863bc Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:40:31 +0300 Subject: [PATCH 159/220] Use token for fork --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index b6a4dcdb4..aa4afb53b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -93,7 +93,7 @@ jobs: id: create-pr uses: peter-evans/create-pull-request@v7 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.gh_token_for_fork }} push-to-fork: Sh-Peter/official-images path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} From 8654ea2aaa7c1c876975b92667c709e1ed2c1d47 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:40:57 +0300 Subject: [PATCH 160/220] Show color diff --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index aa4afb53b..38b31ce21 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -86,7 +86,7 @@ jobs: -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis - cd official-images && git diff + cd official-images && git diff --color cd - - name: Create pull request to official-images From f33f94a667e89adfc9b87a3cfcb5690c9431dcfd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:47:31 +0300 Subject: [PATCH 161/220] Add generated output entries --- release-automation/src/stackbrew_generator/stackbrew.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index d872b65e5..faac43b94 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -113,6 +113,7 @@ def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewE console.print(f"[yellow]No tags generated for {release}[/yellow]") console.print(f"[green]Generated {len(entries)} stackbrew entries[/green]") + console.print(f"[dim]{self.format_stackbrew_output(entries)}[/dim]") return entries def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: From 4faa3a487769d4db05dc6ff421eb658ab67a52be Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:57:29 +0300 Subject: [PATCH 162/220] Upload release_info --- .github/workflows/release_publish.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 38b31ce21..41913426a 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -109,3 +109,21 @@ jobs: run: | echo "Pull Request Number: ${{ steps.create-pr.outputs.pull-request-number }}" echo "Pull Request URL: ${{ steps.create-pr.outputs.pull-request-url }}" + + # Create release_info.json artifact + cat > release_info.json << EOF + { + "pull_request_number": "${{ steps.create-pr.outputs.pull-request-number }}", + "pull_request_url": "${{ steps.create-pr.outputs.pull-request-url }}" + } + EOF + + echo "Created release_info.json:" + cat release_info.json + + - name: Upload release info artifact + uses: actions/upload-artifact@v4 + with: + name: release_info + path: release_info.json + retention-days: 30 From 7f38f4ec2803d25bfbd0cc34b339133e1cce9b8f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 09:59:07 +0300 Subject: [PATCH 163/220] Increase artifact lifetime, remove unused yaml --- .github/workflows/release_build_and_test.yml | 21 +------------------- .github/workflows/release_publish.yml | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 75d8e4ccd..8c232ea8f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -116,23 +116,4 @@ jobs: with: name: release_handle path: release_handle.json - retention-days: 30 - - # - name: Create version tag - # if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} - # uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main - # with: - # tag: v${{ github.event.inputs.release_tag }} - # ref: ${{ steps.merge-back.outputs.merge_commit_sha }} - # gh_token: ${{ secrets.GITHUB_TOKEN }} - - # - name: Test Release Automation Docker Image - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # docker run --rm \ - # -v ${{ github.workspace }}:/workspace \ - # -w /workspace \ - # -e FORCE_COLOR=1 \ - # $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - # generate 8 \ No newline at end of file + retention-days: 400 \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 41913426a..0f6837d43 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -126,4 +126,4 @@ jobs: with: name: release_info path: release_info.json - retention-days: 30 + retention-days: 400 From e7d4b01ec1a74e758ff2ad21fc67f57fb9f1cee5 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 10:07:02 +0300 Subject: [PATCH 164/220] Use redis-developer fork, but target personal repo for testing --- .github/workflows/release_publish.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 0f6837d43..219fafabe 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -12,6 +12,11 @@ on: description: 'Optional UUID to identify this workflow run' required: false +env: + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + FORKED_OFFICIAL_IMAGES_REPO: redis-developer/official-images + # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" @@ -69,7 +74,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Peter-Sh/official-images + repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} - name: Generate stackbrew library content env: @@ -94,7 +99,8 @@ jobs: uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.gh_token_for_fork }} - push-to-fork: Sh-Peter/official-images + draft: true + push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" From d2af09d58364ddbda42b62449034e53c8d5b4742 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:03:27 +0300 Subject: [PATCH 165/220] Use personal token for PR --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 219fafabe..3ca0194d0 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -98,7 +98,7 @@ jobs: id: create-pr uses: peter-evans/create-pull-request@v7 with: - token: ${{ secrets.gh_token_for_fork }} + token: ${{ secrets.GH_TOKEN_FOR_PR }} draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images From 800abcba816cd0c91cf36180f35d077b6c38d4d9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:10:52 +0300 Subject: [PATCH 166/220] Fix fork url --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 3ca0194d0..f1fe15395 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -15,7 +15,7 @@ on: env: #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images - FORKED_OFFICIAL_IMAGES_REPO: redis-developer/official-images + FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 2de5de1fe61cf45a87d600ee0cba74c054e6ea6f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:28:26 +0300 Subject: [PATCH 167/220] Add a DRAFT note --- .github/workflows/release_publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f1fe15395..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -106,6 +106,9 @@ jobs: commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | + THIS IS A DRAFT PR FOR AUTOMATION TESTING + PLEASE IGNORE AND SORRY FOR BOTHERING + Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} From 333a41b3adfdbe17bfed4a61480102b4748d5b54 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:02:09 +0300 Subject: [PATCH 168/220] Try syncing fork and pushing to master --- .github/workflows/release_publish.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..a3bd66969 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -76,6 +76,16 @@ jobs: path: official-images repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} + - name: Sync fork with upstream + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.gh_token_for_fork }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${{ env.FORKED_OFFICIAL_IMAGES_REPO }}/merge-upstream \ + -d '{"branch":"master"}' + - name: Generate stackbrew library content env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -102,12 +112,14 @@ jobs: draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images - branch: redis-${{ steps.parse-release.outputs.release_version }} + branch: master + base: master + sign-commits: true commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR BOTHERING + PLEASE IGNORE AND SORRY FOR THE BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} From 061bc6b3c5567fda0b1d9d98bfe15cc5e01d64f2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:06:16 +0300 Subject: [PATCH 169/220] Revert "Try syncing fork and pushing to master" This reverts commit 8d99774d1c2c7b8d0f671353fda14207aa5da4f7. --- .github/workflows/release_publish.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index a3bd66969..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -76,16 +76,6 @@ jobs: path: official-images repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} - - name: Sync fork with upstream - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.gh_token_for_fork }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{ env.FORKED_OFFICIAL_IMAGES_REPO }}/merge-upstream \ - -d '{"branch":"master"}' - - name: Generate stackbrew library content env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -112,14 +102,12 @@ jobs: draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images - branch: master - base: master - sign-commits: true + branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR THE BOTHERING + PLEASE IGNORE AND SORRY FOR BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} From ad3f9b2a384ac79f4582f50754a402ec35a67df4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:14:14 +0300 Subject: [PATCH 170/220] Use official docker for testing --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..e551f8584 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 35a31805df9afa2d245eb3457520645209bd398c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 15:39:28 +0300 Subject: [PATCH 171/220] Collect image urls to release handle --- .../actions/build-and-tag-locally/action.yml | 24 +++++++++++- .github/workflows/pre-merge.yml | 37 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 25 ++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 060384d5b..a632dccff 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -1,3 +1,4 @@ + name: Build and Test inputs: @@ -231,11 +232,12 @@ runs: id: format-registry-tag shell: bash run: | - printf "tag=%s:%s%s-%s" \ + printf "tag=%s:%s%s-%s-%s" \ "${{ inputs.registry_repository }}" \ "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \ "${{ github.sha }}" \ "${{ inputs.distribution }}" \ + "${{ steps.platform.outputs.display_name }}" \ | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" - name: Push image @@ -247,3 +249,23 @@ runs: tags: ${{ steps.format-registry-tag.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max + + - name: Save image URL to artifact + shell: bash + run: | + if [[ "${{ inputs.publish_image }}" == "true" && "${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}" == "true" ]]; then + # Create a file with the image URL for this specific build + mkdir -p /tmp/image-urls + echo "${{ steps.format-registry-tag.outputs.tag }}" > "/tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt" + echo "Image URL saved: ${{ steps.format-registry-tag.outputs.tag }}" + else + echo "Image not published for this platform/distribution combination" + fi + + - name: Upload image URL artifact + uses: actions/upload-artifact@v4 + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + with: + name: image-url-${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + path: /tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt + retention-days: 1 diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 0924c7eb3..94b0246a9 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -10,6 +10,10 @@ on: description: 'Release tag to build' required: true type: string + outputs: + docker_image_urls: + description: 'Array of Docker image URLs that were published' + value: ${{ jobs.collect-image-urls.outputs.docker_image_urls }} jobs: build-and-test: @@ -59,3 +63,36 @@ jobs: registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} + collect-image-urls: + runs-on: ubuntu-latest + needs: build-and-test + if: ${{ inputs.release_tag }} + outputs: + docker_image_urls: ${{ steps.collect-urls.outputs.urls }} + steps: + - name: Download all image URL artifacts + uses: actions/download-artifact@v4 + with: + pattern: image-url-* + path: ./image-urls + merge-multiple: true + + - name: Collect image URLs from artifacts + id: collect-urls + run: | + # Create JSON array from all URL files using jq + if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then + echo "Found image URL files:" + ls -la ./image-urls/ + + # Read all URL files and create JSON array with jq + urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') + + echo "Collected image URLs: $urls" + else + echo "No image URL artifacts found" + urls="[]" + fi + + echo "urls=$urls" >> "$GITHUB_OUTPUT" + diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 8c232ea8f..b4ac6a29a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -52,16 +52,11 @@ jobs: release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: - runs-on: ["ubuntu-latest"] - steps: - - shell: bash - run: | - echo Hi + uses: ./.github/workflows/pre-merge.yml needs: prepare-release - #uses: ./.github/workflows/pre-merge.yml - #secrets: inherit - #with: - # release_tag: ${{ github.event.inputs.release_tag }} + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] @@ -99,12 +94,22 @@ jobs: exit 1 fi + # Get docker image URLs from build-and-test job + DOCKER_IMAGE_URLS='${{ needs.build-and-test.outputs.docker_image_urls }}' + + # Validate that DOCKER_IMAGE_URLS is valid JSON + if ! echo "$DOCKER_IMAGE_URLS" | jq . > /dev/null 2>&1; then + echo "Warning: docker_image_urls is not valid JSON, using empty array" + DOCKER_IMAGE_URLS="[]" + fi + cat > release_handle.json << EOF { "release_commit_sha": "$RELEASE_COMMIT_SHA", "release_version": "${{ github.event.inputs.release_tag }}", "release_version_branch": "${{ steps.ensure-branch.outputs.release_version_branch }}", - "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}" + "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}", + "docker_image_urls": $DOCKER_IMAGE_URLS } EOF From 9c2abfe00e8c5875455fafba1a5213b8dee84428 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 15:41:28 +0300 Subject: [PATCH 172/220] Return to personal repo for testing --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index e551f8584..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From ab49c2b04bd86661c2fd2a0cc06f7044789aac5d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 17:36:31 +0300 Subject: [PATCH 173/220] Temporary disable some of the architectures --- .github/workflows/pre-merge.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 94b0246a9..067302cfb 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -1,3 +1,4 @@ + name: Build and Test on: pull_request: @@ -26,15 +27,15 @@ jobs: - alpine platform: - linux/amd64 - - linux/i386 - - linux/arm/v5 - - linux/arm/v6 - - linux/arm/v7 - - linux/mips64le - - linux/ppc64le - - linux/s390x + # - linux/i386 + # - linux/arm/v5 + # - linux/arm/v6 + # - linux/arm/v7 + # - linux/mips64le + # - linux/ppc64le + # - linux/s390x - linux/arm64 - - linux/riscv64 + # - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -80,14 +81,9 @@ jobs: - name: Collect image URLs from artifacts id: collect-urls run: | - # Create JSON array from all URL files using jq if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then echo "Found image URL files:" - ls -la ./image-urls/ - - # Read all URL files and create JSON array with jq urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') - echo "Collected image URLs: $urls" else echo "No image URL artifacts found" @@ -95,4 +91,3 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" - From 688f9597dfa120a8799649006b04938d3b2b5115 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 17:43:23 +0300 Subject: [PATCH 174/220] Leave only 2 images --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 067302cfb..d9aa36b70 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -24,7 +24,7 @@ jobs: matrix: distribution: - debian - - alpine + # - alpine platform: - linux/amd64 # - linux/i386 From b997ae6ef2122eedc1b77a08f58ab42a21b88a8d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 18:25:45 +0300 Subject: [PATCH 175/220] Try self-hosted runners --- .github/workflows/pre-merge.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index d9aa36b70..376d965b9 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -18,7 +18,8 @@ on: jobs: build-and-test: - runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} + #runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} + runs-on: ["${{ contains(matrix.platform, 'arm64') && 'ARM64' || 'X64' }}", "self-hosted"] strategy: fail-fast: false matrix: From a7deb847c611575238e74d9efb89bead022f210c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 20:35:01 +0300 Subject: [PATCH 176/220] Fix image urls --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 376d965b9..6866e7c17 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -84,7 +84,7 @@ jobs: run: | if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then echo "Found image URL files:" - urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') + urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s -c 'split("\n") | map(select(length > 0))') echo "Collected image URLs: $urls" else echo "No image URL artifacts found" From c5fa85def853fe2a194e2c75a33013c514a2d5db Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 20:45:02 +0300 Subject: [PATCH 177/220] Used scoped layer cache --- .github/actions/build-and-tag-locally/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index a632dccff..d1e7a67ee 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -99,8 +99,8 @@ runs: load: true platforms: ${{ inputs.platform }} tags: ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + cache-to: type=gha,mode=max,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} - name: Save image shell: bash From a3a372be2757afa9a8bc8fb8e2c68e83e84fa624 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:24:55 +0300 Subject: [PATCH 178/220] Introduce wait for redis instead of always sleep --- test/run-entrypoint-tests.sh | 69 ++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 28d138ae0..5cfeff020 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -41,7 +41,6 @@ get_container_user_uid_gid_on_the_host() { container_user="$1" dir=$(mktemp -d -p .) docker run --rm -v "$(pwd)/$dir":/w -w /w --entrypoint=/bin/sh "$REDIS_IMG" -c "chown $container_user ." - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize stat -c "%u %g" "$dir" sudo rm -rf "$dir" } @@ -56,6 +55,42 @@ fi # Helper functions # +# Wait for Redis server or sentiel to be ready in a container by pinging it +# Arguments: +# $1 - container name/id +# Returns: +# 0 if Redis responds with PONG within timeout +# 1 if timeout CONTAINER_INIT_WAIT occurs +wait_for_redis_server_in_container() { + local container="$1" + local timeout="${CONTAINER_INIT_WAIT:-3}" + local elapsed=0 + local sleep_interval=0.1 + + if [ -z "$container" ]; then + return 1 + fi + + while [[ "$elapsed" < "$timeout" ]]; do + # Try to ping Redis server + if response=$(docker exec "$container" redis-cli ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + if response=$(docker exec "$container" redis-cli -p 26379 ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + # Sleep and increment elapsed time + sleep "$sleep_interval" + elapsed=$(awk "BEGIN {print $elapsed + $sleep_interval}") + done + + echo "Timeout: Redis server did not respond within ${timeout}s" + docker stop "$container" >/dev/null + return 1 +} + # creates one entry of directory structure # used in combination with iterate_dir_structure_with create_entry() { @@ -181,7 +216,6 @@ run_docker_and_test_ownership() { fi docker_output=$($docker_run 2>&1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize if [ "$TEST_VERBOSE" ]; then echo "After:" @@ -270,14 +304,9 @@ run_redis_docker_and_check_uid_gid() { docker_cmd="$*" # shellcheck disable=SC2086 container=$(docker run $docker_flags -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize ret=$? - assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - if [ $ret -gt 0 ]; then - echo "retarning" - return 1 - fi + wait_for_redis_server_in_container "$container" || return 1 cmdline=$(docker exec "$container" cat /proc/1/cmdline|tr -d \\0) assertContains "$docker_flags $docker_cmd, cmdline: $cmdline" "$cmdline" "$expected_cmd" @@ -302,7 +331,10 @@ run_redis_docker_and_check_modules() { docker_cmd="$1" # shellcheck disable=SC2086 container=$(docker run --rm -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + info=$(docker exec "$container" redis-cli info) [ "$PLATFORM" ] && [ "$PLATFORM" != "amd64" ] && startSkipping @@ -329,7 +361,6 @@ assert_redis_v8() { test_redis_version() { ret=$(docker run --rm "$REDIS_IMG" -v|tail -n 1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize assert_redis_v8 "$ret" } @@ -553,7 +584,9 @@ test_redis_server_persistence_with_bind_mount() { chmod 0444 "$dir" container=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -568,7 +601,10 @@ test_redis_server_persistence_with_bind_mount() { sudo chown -R "$HOST_OWNER" "$dir" container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -586,7 +622,9 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data' container=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -601,7 +639,10 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data && chmod 0000 -R /data' container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" From f5d5e905098266e46f105115faaae1cf9a3a8980 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:32:33 +0300 Subject: [PATCH 179/220] Increase timeout --- test/run-entrypoint-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 5cfeff020..264466aad 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -63,7 +63,7 @@ fi # 1 if timeout CONTAINER_INIT_WAIT occurs wait_for_redis_server_in_container() { local container="$1" - local timeout="${CONTAINER_INIT_WAIT:-3}" + local timeout="${CONTAINER_INIT_WAIT:-6}" local elapsed=0 local sleep_interval=0.1 From 9536d72f447e932cb460b50bc44664d27c72c7db Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:36:23 +0300 Subject: [PATCH 180/220] Really increase timeout --- test/run-entrypoint-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 264466aad..7bffbc4ba 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -21,7 +21,7 @@ ## # Container initialization wait time in seconds -CONTAINER_INIT_WAIT=3 +CONTAINER_INIT_WAIT=6 if [ -z "$REDIS_IMG" ]; then echo "REDIS_IMG may not be empty" @@ -63,7 +63,7 @@ fi # 1 if timeout CONTAINER_INIT_WAIT occurs wait_for_redis_server_in_container() { local container="$1" - local timeout="${CONTAINER_INIT_WAIT:-6}" + local timeout="${CONTAINER_INIT_WAIT:-3}" local elapsed=0 local sleep_interval=0.1 From 47d5c1bc4b8188ee5b0d66427ae9395f5b7ad8cc Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:44:22 +0300 Subject: [PATCH 181/220] Fix wait for redis: use correct container id --- test/run-entrypoint-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 7bffbc4ba..4516c6c9d 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -21,7 +21,7 @@ ## # Container initialization wait time in seconds -CONTAINER_INIT_WAIT=6 +CONTAINER_INIT_WAIT=3 if [ -z "$REDIS_IMG" ]; then echo "REDIS_IMG may not be empty" @@ -603,7 +603,7 @@ test_redis_server_persistence_with_bind_mount() { container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") ret=$? assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - wait_for_redis_server_in_container "$container" || return 1 + wait_for_redis_server_in_container "$container2" || return 1 value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -641,7 +641,7 @@ test_redis_server_persistence_with_volume() { container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") ret=$? assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - wait_for_redis_server_in_container "$container" || return 1 + wait_for_redis_server_in_container "$container2" || return 1 value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" From c61a6ea646af20e0d961cef46494b49d63668d1f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:53:23 +0300 Subject: [PATCH 182/220] Return official images for demo --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..e551f8584 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 5a34e94e026f4a91d33cd32793a2e2075301a7a7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 11:31:39 +0300 Subject: [PATCH 183/220] Add slack notification about image urls --- .github/workflows/pre-merge.yml | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6866e7c17..6e75785ea 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,3 +92,55 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" + + notify-slack: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && needs.collect-image-urls.outputs.docker_image_urls != '[]' }} + steps: + - name: Send Slack notification + run: | + # Parse the image URLs from JSON array + image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' + + # Create formatted list of image URLs + formatted_urls=$(echo "$image_urls" | jq -r '.[] | "• `" + . + "`"' | tr '\n' '\n') + + # Create Slack message payload + cat > slack_payload.json << EOF + { + "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "The following Docker images have been successfully published:\n\n$formatted_urls" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + } + ] + } + ] + } + EOF + + # Send to Slack + curl -X POST -H 'Content-type: application/json' \ + --data @slack_payload.json \ + "${{ secrets.SLACK_WEB_HOOK_URL }}" + + echo "Slack notification sent for release ${{ inputs.release_tag }}" From af9ca7f97e0af14b53c11ae5639047ad1526404c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 11:31:56 +0300 Subject: [PATCH 184/220] Revert "Return official images for demo" This reverts commit 0dd060d69308a6c683beb095838a699f5dcc7fa9. --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index e551f8584..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From c51ce0d8186da96133657019539b99aaad298e0f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:13:46 +0300 Subject: [PATCH 185/220] Reformat slack message --- .github/actions/common/func.sh | 43 ++++++++++++++++++++++++++++++ .github/workflows/pre-merge.yml | 46 ++++----------------------------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index efb4ccee3..edeea426f 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -330,4 +330,47 @@ prepare_releases_list() { increase_indent_level console_output 2 gray "$debug_output" decrease_indent_level +} + +slack_format_docker_image_urls_message() { + # Parse the image URLs from JSON array + local image_urls formatted_urls release_tag footer + image_urls=$(cat) + release_tag=$1 + footer=$2 + + # Create formatted list of image URLs + formatted_urls=$(echo "$image_urls" | jq -j '.[] | "\\n• \(.)"') + +# Create Slack message payload + cat << EOF +{ +"text": "🐳 Docker Images Published for Release $release_tag", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Images Published for Release $release_tag" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "The following Docker images have been successfully published:\n\n$formatted_urls" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "f$footer" + } + ] + } +] +} +EOF } \ No newline at end of file diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6e75785ea..e49fbcb81 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -100,47 +100,11 @@ jobs: steps: - name: Send Slack notification run: | - # Parse the image URLs from JSON array image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' + workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - # Create formatted list of image URLs - formatted_urls=$(echo "$image_urls" | jq -r '.[] | "• `" + . + "`"' | tr '\n' '\n') + . .github/actions/common/func.sh - # Create Slack message payload - cat > slack_payload.json << EOF - { - "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The following Docker images have been successfully published:\n\n$formatted_urls" - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" - } - ] - } - ] - } - EOF - - # Send to Slack - curl -X POST -H 'Content-type: application/json' \ - --data @slack_payload.json \ - "${{ secrets.SLACK_WEB_HOOK_URL }}" - - echo "Slack notification sent for release ${{ inputs.release_tag }}" + echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ + curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 18673cc69fc286c2c0f650ef80427ad34f13a03f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:26:42 +0300 Subject: [PATCH 186/220] Slack msg for PR creation --- .github/actions/common/func.sh | 42 +++++++++++++++++++++++++-- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 10 +++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index edeea426f..48a974c01 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -345,7 +345,7 @@ slack_format_docker_image_urls_message() { # Create Slack message payload cat << EOF { -"text": "🐳 Docker Images Published for Release $release_tag", +"text": "🐳 Docker Images Published for Redis: $release_tag", "blocks": [ { "type": "header", @@ -366,7 +366,45 @@ slack_format_docker_image_urls_message() { "elements": [ { "type": "mrkdwn", - "text": "f$footer" + "text": "$footer" + } + ] + } +] +} +EOF +} + +slack_format_docker_PR_message() { + release_tag=$1 + url=$2 + footer=$3 + +# Create Slack message payload + cat << EOF +{ +"text": "🐳 Docker Library PR created for Redis: $release_tag", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Library PR created for Redis: $release_tag" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "$url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" } ] } diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index e49fbcb81..c2af8345f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -104,7 +104,7 @@ jobs: workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . .github/actions/common/func.sh + . ${{ github.workspace}}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..2ac602468 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -136,3 +136,13 @@ jobs: name: release_info path: release_info.json retention-days: 400 + + - name: Send Slack notification + run: | + workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + + . ${{ github.workspace }}/.github/actions/common/func.sh + + slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file From 13bacc07bb7d49c785a79b647a5aceefe7ee11c4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:39:26 +0300 Subject: [PATCH 187/220] Debug output --- .github/workflows/release_build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b4ac6a29a..8a121b278 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,14 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Test include + run: | + set -x + pwd + ls -la ${GITHUB_WORKSPACE} + find ${GITHUB_WORKSPACE} -name .github + exit 1 + - name: Log in to Container Registry uses: docker/login-action@v3 with: From b4675ca9f5c456b864e334174e2a380c8efb2e5d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:42:29 +0300 Subject: [PATCH 188/220] Further debug --- .github/workflows/release_build_and_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 8a121b278..6cd8dafa7 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -30,6 +30,9 @@ jobs: pwd ls -la ${GITHUB_WORKSPACE} find ${GITHUB_WORKSPACE} -name .github + find ${GITHUB_WORKSPACE} -name func.sh + echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh exit 1 - name: Log in to Container Registry From 5db086416bbe82ef70749453ee762772c3f272ce Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:46:35 +0300 Subject: [PATCH 189/220] Try using GITHUB_WORKSPACE env --- .github/workflows/pre-merge.yml | 7 ++++++- .github/workflows/release_build_and_test.yml | 11 ----------- .github/workflows/release_publish.yml | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index c2af8345f..5aa33d5ad 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -104,7 +104,12 @@ jobs: workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . ${{ github.workspace}}/.github/actions/common/func.sh + pwd + ls -la ${GITHUB_WORKSPACE} + find ${GITHUB_WORKSPACE} -name .github + find ${GITHUB_WORKSPACE} -name func.sh + echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6cd8dafa7..b4ac6a29a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,17 +24,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Test include - run: | - set -x - pwd - ls -la ${GITHUB_WORKSPACE} - find ${GITHUB_WORKSPACE} -name .github - find ${GITHUB_WORKSPACE} -name func.sh - echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - exit 1 - - name: Log in to Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 2ac602468..f5235941b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -142,7 +142,7 @@ jobs: workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . ${{ github.workspace }}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file From fa7c8ebb3b1f7412d798fd307e366adc8b8931a4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:55:33 +0300 Subject: [PATCH 190/220] Use checkout action --- .github/workflows/pre-merge.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 5aa33d5ad..c8b354868 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -98,17 +98,15 @@ jobs: needs: collect-image-urls if: ${{ inputs.release_tag && needs.collect-image-urls.outputs.docker_image_urls != '[]' }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Send Slack notification run: | image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - pwd - ls -la ${GITHUB_WORKSPACE} - find ${GITHUB_WORKSPACE} -name .github - find ${GITHUB_WORKSPACE} -name func.sh - echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ From 26177aeb89d5b045b5af86c8957fde78cae4ffb5 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 13:04:53 +0300 Subject: [PATCH 191/220] Fix curl execution for slack --- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index c8b354868..4d2ebc712 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -110,4 +110,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ - curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f5235941b..2061aff9c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -145,4 +145,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From fa6360b3ba762165fbb53c69bf34864aca65f98c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 13:05:52 +0300 Subject: [PATCH 192/220] Use official repo --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 2061aff9c..c90cb76e3 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 164c1240dcc6b5530281d1d86e74a2de0554b19a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 20:45:31 +0300 Subject: [PATCH 193/220] Proper milestone filtering and tests --- release-automation/README.md | 16 +- .../src/stackbrew_generator/models.py | 10 + .../src/stackbrew_generator/version_filter.py | 34 +- .../tests/test_stackbrew_updater.py | 123 +++++++ .../tests/test_update_stackbrew_file.py | 297 ++++++++++++++++ .../tests/test_version_filter.py | 323 ++++++++++++++++++ 6 files changed, 787 insertions(+), 16 deletions(-) create mode 100644 release-automation/tests/test_stackbrew_updater.py create mode 100644 release-automation/tests/test_update_stackbrew_file.py create mode 100644 release-automation/tests/test_version_filter.py diff --git a/release-automation/README.md b/release-automation/README.md index e37e7189a..3e07a6d1e 100644 --- a/release-automation/README.md +++ b/release-automation/README.md @@ -16,13 +16,27 @@ Each version release is tagged with `vMajor.Minor.Patch` (e.g. `v8.2.1`) Milestone releases are tagged with `vMajor.Minor.Patch-Milestone` (e.g. `v8.2.1-m01`). Any suffix after patch version is considered a milestone. +Suffixes starting with `rc` are considered release candidates and are preferred over suffixes starting with `m` which in turn are preferred over any other suffix. + Tags without suffix are considered GA (General Availability) releases (e.g. `v8.2.1`). Internal releases are milestone releases containing `-int` in their name (e.g. `v8.2.1-m01-int1` or `8.4.0-int3`). They are not released to the public. Milestone releases never get latest or any other default tags, like `8`, `8.2`, `8.2.1`, `latest`, `bookworm`, etc. -For each mainline only one GA release and optionally one milestone release should be published in official-library. The most latest versions. +For each mainline only one GA release and optionally any number of milestone releases with higher versions than this GA may be published in official-library. + +Each patch version may have only one GA or milestone release, GA release is preferred over milestone release. + +For example for this list of tags the following rules will be applied + +* `v8.2.3-m01` - included because there is neither GA nor any higher milestone versions for 8.2.3 +* `v8.2.2-rc2` - included because it higher version among 8.2.2 +* `v8.2.2-rc1` - excluded because 8.2.2-rc2 is higher version +* `v8.2.2-m01` - excluded because 8.2.2-rc2 is higher version +* `v8.2.1-rc2` - excluded because there is 8.2.1 GA version +* `v8.2.1` - included because it is highest GA for 8.2 +* `v8.2.0` - exluded because 8.2.1 is higher version End of life versions are marked with `-eol` suffix (e.g. `v8.0.3-eol`). When there is a at least one minor version tagged with eol all versions in this minor series are considered EOL and are not included in the release file. diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 6d7c88991..8a9a1fff7 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -68,6 +68,16 @@ def mainline_version(self) -> str: """Get the mainline version string (major.minor).""" return f"{self.major}.{self.minor}" + @property + def sort_key(self) -> str: + suffix_weight = 0 + if self.suffix.startswith("rc"): + suffix_weight = 100 + elif self.suffix.startswith("m"): + suffix_weight = 50 + + return f"{self.major}.{self.minor}.{self.patch or 0}.{suffix_weight}.{self.suffix}" + def __str__(self) -> str: """String representation of the version.""" version = f"{self.major}.{self.minor}" diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index 23cb0ed3b..13db27443 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -2,6 +2,8 @@ from typing import Dict, List, Tuple +from collections import OrderedDict + from packaging.version import Version from rich.console import Console @@ -47,11 +49,12 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe continue # Sort by version (newest first) - versions.sort(key=lambda x: x[0], reverse=True) + versions.sort(key=lambda x: x[0].sort_key, reverse=True) console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") return versions + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter out end-of-life versions. @@ -99,24 +102,25 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str, str]]) """ console.print("[blue]Filtering to actual versions (latest patch per minor/milestone)[/blue]") - seen_combinations = set() - filtered_versions = [] + patch_versions = OrderedDict() for version, commit, tag_ref in versions: - # Create a key for minor version + milestone status - combination_key = (version.mainline_version, version.is_milestone) + patch_key = (version.major, version.minor, version.patch) + if patch_key not in patch_versions: + patch_versions[patch_key] = (version, commit, tag_ref) + elif patch_versions[patch_key][0].is_milestone and not version.is_milestone: + # GA always takes precedence over milestone for the same major.minor.patch + patch_versions[patch_key] = (version, commit, tag_ref) + + print(patch_versions.values()) + filtered_versions = [] + mainlines_with_ga = set() - if combination_key not in seen_combinations: - seen_combinations.add(combination_key) + for version, commit, tag_ref in patch_versions.values(): + if version.mainline_version not in mainlines_with_ga: + if not version.is_milestone: + mainlines_with_ga.add(version.mainline_version) filtered_versions.append((version, commit, tag_ref)) - - milestone_str = "milestone" if version.is_milestone else "GA" - console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") - else: - milestone_str = "milestone" if version.is_milestone else "GA" - console.print(f"[dim]Skipping {version} ({milestone_str}) - already have this minor/milestone combination[/dim]") - - console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") return filtered_versions def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: diff --git a/release-automation/tests/test_stackbrew_updater.py b/release-automation/tests/test_stackbrew_updater.py new file mode 100644 index 000000000..36b98590e --- /dev/null +++ b/release-automation/tests/test_stackbrew_updater.py @@ -0,0 +1,123 @@ +"""Tests for StackbrewUpdater class.""" + +import tempfile +from pathlib import Path + +from stackbrew_generator.stackbrew import StackbrewUpdater + +class TestStackbrewUpdater: + """Tests for StackbrewUpdater class.""" + + def setup_method(self): + """Set up test fixtures.""" + self.updater = StackbrewUpdater() + + def test_update_stackbrew_content_basic(self): + """Test basic stackbrew content update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + new_content = """Tags: 8.2.2, 8.2, 8, 8.2.2-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: debian + +Tags: 8.2.2-alpine, 8.2-alpine, 8-alpine, 8.2.2-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: alpine""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + # Update the content + updated_content = self.updater.update_stackbrew_content( + input_file, 8, new_content, verbose=False + ) + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_parse_stackbrew_entries(self): + """Test parsing stackbrew entries.""" + lines = [ + "Tags: 8.2.1, 8.2, 8", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: debian", + "", + "Tags: 8.2.1-alpine, 8.2-alpine", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: alpine" + ] + + entries = self.updater._parse_stackbrew_entries(lines) + + assert len(entries) == 2 + assert entries[0][0] == "Tags: 8.2.1, 8.2, 8" + assert entries[1][0] == "Tags: 8.2.1-alpine, 8.2-alpine" + + def test_entry_belongs_to_major_version(self): + """Test checking if entry belongs to major version.""" + entry_8x = [ + "Tags: 8.2.1, 8.2, 8, latest", + "Architectures: amd64", + "GitCommit: abc123", + "Directory: debian" + ] + + entry_7x = [ + "Tags: 7.4.0, 7.4, 7", + "Architectures: amd64", + "GitCommit: def456", + "Directory: debian" + ] + + assert self.updater._entry_belongs_to_major_version(entry_8x, 8) is True + assert self.updater._entry_belongs_to_major_version(entry_8x, 7) is False + assert self.updater._entry_belongs_to_major_version(entry_7x, 7) is True + assert self.updater._entry_belongs_to_major_version(entry_7x, 8) is False diff --git a/release-automation/tests/test_update_stackbrew_file.py b/release-automation/tests/test_update_stackbrew_file.py new file mode 100644 index 000000000..acdbd3ac1 --- /dev/null +++ b/release-automation/tests/test_update_stackbrew_file.py @@ -0,0 +1,297 @@ +"""Tests for update-stackbrew-file command.""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from typer.testing import CliRunner + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release + + +class TestUpdateStackbrewFile: + """Tests for update-stackbrew-file command.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_update_stackbrew_file_basic(self): + """Test basic stackbrew file update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Mock git client + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + # Mock distribution detector + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + # Mock version filter + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + + # Mock the version filter to return Redis 8.x versions + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.2"), "new123commit", "refs/tags/v8.2.2") + ] + + # Mock releases + mock_releases = [ + Release( + commit="new123commit", + version=RedisVersion.parse("8.2.2"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.2" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + # Run the command with output to file + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(input_file), + "--verbose" + ]) + + assert result.exit_code == 0 + + # Check that the file was updated + updated_content = input_file.read_text() + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_update_stackbrew_file_nonexistent_input(self): + """Test error handling for nonexistent input file.""" + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", "/nonexistent/file.txt" + ]) + + assert result.exit_code == 1 + assert "Input file does not exist" in result.stderr + + def test_update_stackbrew_file_no_versions_found(self): + """Test error handling when no versions are found.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [] + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "9", + "--input", str(input_file) + ]) + + assert result.exit_code == 1 + assert "No versions found for Redis 9.x" in result.stderr + + finally: + input_file.unlink() + + def test_update_stackbrew_file_with_output_option(self): + """Test using separate output file.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as input_f, \ + tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as output_f: + + input_f.write(sample_content) + input_file = Path(input_f.name) + output_file = Path(output_f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new456commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new456commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(output_file) + ]) + + assert result.exit_code == 0 + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + # Output file should have updated content + updated_content = output_file.read_text() + assert "new456commit" in updated_content + assert "8.2.0" in updated_content + assert "old123" not in updated_content + + finally: + input_file.unlink() + output_file.unlink() + + def test_update_stackbrew_file_stdout_output(self): + """Test outputting to stdout when no output file is specified.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new789commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new789commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file) + ]) + + assert result.exit_code == 0 + + # Should output to stdout + assert "new789commit" in result.stdout + assert "8.2.0" in result.stdout + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + finally: + input_file.unlink() diff --git a/release-automation/tests/test_version_filter.py b/release-automation/tests/test_version_filter.py new file mode 100644 index 000000000..7f704da28 --- /dev/null +++ b/release-automation/tests/test_version_filter.py @@ -0,0 +1,323 @@ +"""Tests for VersionFilter class.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.models import RedisVersion +from stackbrew_generator.version_filter import VersionFilter +from stackbrew_generator.git_operations import GitClient +from stackbrew_generator.exceptions import GitOperationError + + +class MockGitClient: + """Mock GitClient for testing.""" + + def __init__(self): + """Initialize mock git client.""" + self.remote_tags = [] + self.version_extraction_results = {} + + def set_remote_tags(self, tags): + """Set mock remote tags. + + Args: + tags: List of (commit, tag_ref) tuples + """ + self.remote_tags = tags + + def set_version_extraction_result(self, tag_ref, version_or_exception): + """Set mock version extraction result. + + Args: + tag_ref: Tag reference + version_or_exception: RedisVersion instance or Exception to raise + """ + self.version_extraction_results[tag_ref] = version_or_exception + + def list_remote_tags(self, major_version): + """Mock list_remote_tags method.""" + return self.remote_tags + + def extract_version_from_tag(self, tag_ref, major_version): + """Mock extract_version_from_tag method.""" + if tag_ref in self.version_extraction_results: + result = self.version_extraction_results[tag_ref] + if isinstance(result, Exception): + raise result + return result + # Default behavior - try to parse from tag_ref + return RedisVersion.parse(tag_ref.replace('refs/tags/', '')) + + +def create_version_tuples(version_strings): + """Helper to create version tuples from version strings. + + Args: + version_strings: List of version strings + + Returns: + List of (RedisVersion, commit, tag_ref) tuples + """ + tuples = [] + for i, version_str in enumerate(version_strings): + version = RedisVersion.parse(version_str) + commit = f"commit{i:03d}" + tag_ref = f"refs/tags/{version_str}" + tuples.append((version, commit, tag_ref)) + + tuples.sort(key=lambda x: x[0].sort_key, reverse=True) + return tuples + + +class TestVersionFilter: + """Tests for VersionFilter class.""" + + def test_init(self): + """Test VersionFilter initialization.""" + git_client = GitClient() + version_filter = VersionFilter(git_client) + assert version_filter.git_client is git_client + + def test_get_redis_versions_from_tags_success(self): + """Test successful version retrieval from tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should be sorted by version (newest first) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.1.0"] + assert version_strings == expected_versions + + # Check commits and tag refs + commits = [v[1] for v in result] + tag_refs = [v[2] for v in result] + expected_commits = ["commit001", "commit002", "commit003"] + expected_tag_refs = ["refs/tags/v8.2.1", "refs/tags/v8.2.0", "refs/tags/v8.1.0"] + assert commits == expected_commits + assert tag_refs == expected_tag_refs + + def test_get_redis_versions_from_tags_with_invalid_tags(self): + """Test version retrieval with some invalid tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/invalid-tag"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + # Set up invalid tag to raise exception + mock_git_client.set_version_extraction_result( + "refs/tags/invalid-tag", + ValueError("Invalid version format") + ) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should skip invalid tag and return only valid ones + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_get_redis_versions_from_tags_empty(self): + """Test version retrieval with no tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + assert result == [] + + def test_filter_eol_versions_basic(self): + """Test basic EOL version filtering.""" + version_filter = VersionFilter(MockGitClient()) + + # Create test versions with one EOL minor version + versions = create_version_tuples([ + "v8.2.1", + "v8.2.0", + "v8.1.0-eol", + "v8.1.2", + "v8.0.1", + "v8.0.0" + ]) + + result = version_filter.filter_eol_versions(versions) + + # Should filter out all 8.1.* versions (because 8.1.0-eol exists) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.0.1", "8.0.0"] + assert version_strings == expected_versions + + def test_filter_eol_versions_empty(self): + """Test EOL filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_eol_versions([]) + assert result == [] + + def test_filter_actual_versions_basic(self): + """Test basic actual version filtering (latest patch per minor/milestone).""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with multiple patches for same minor version + versions = create_version_tuples([ + "v8.2.2", # Latest patch for 8.2 GA + "v8.2.1", # Older patch for 8.2 GA + "v8.2.0", # Oldest patch for 8.2 GA + "v8.1.1", # Latest patch for 8.1 GA + "v8.1.0", # Older patch for 8.1 GA + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep only latest patch for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2", "8.1.1"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_same_patch(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version + "v8.2.1-m02", # Latest milestone for 8.2 + "v8.2.1-m01", # Older milestone for 8.2 + "v8.1.0", # GA version + "v8.1.0-m01", # Milestone for 8.1 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_mainline(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + "v8.1.1-m01", # Milestone for 8.1 + "v8.2.0-m03", # Older milestone for 8.2.0 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-m02", "8.2.1", "8.1.1-m01", "8.1.0"] + assert version_strings == expected_versions + + def test_when_filter_actual_versions_with_milestones_rc_is_preferred(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-rc01", # Latest milestone for 8.2.2 + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-rc01", "8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_milestone_only(self): + """Test actual version filtering with only milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + versions = create_version_tuples([ + "v8.2.1-m02", + "v8.2.1-m01", + "v8.1.0-m01", + ]) + + result = version_filter.filter_actual_versions(versions) + + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1-m02", "8.1.0-m01"] + assert version_strings == expected_versions + + def test_filter_actual_versions_empty(self): + """Test actual version filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_actual_versions([]) + assert result == [] + + def test_get_actual_major_redis_versions_success(self): + """Test the main entry point method with successful flow.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0-eol"), # Should be filtered out + ("commit004", "refs/tags/v8.0.1"), + ("commit005", "refs/tags/v8.0.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + # Should apply all filters: get tags -> filter EOL -> filter actual + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.0.1"] # Latest patches, no EOL + assert version_strings == expected_versions + + def test_get_actual_major_redis_versions_no_versions(self): + """Test main entry point with no versions found.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + assert result == [] + +class TestVersionFilterIntegration: + """Integration tests using real GitClient (mocked at subprocess level).""" + + @patch('stackbrew_generator.git_operations.subprocess.run') + def test_integration_with_real_git_client(self, mock_subprocess): + """Test VersionFilter with real GitClient (mocked subprocess).""" + # Mock git ls-remote output + mock_subprocess.return_value.stdout = ( + "commit001\trefs/tags/v8.2.1\n" + "commit002\trefs/tags/v8.2.0\n" + "commit003\trefs/tags/v8.1.0-eol\n" + ) + mock_subprocess.return_value.returncode = 0 + + git_client = GitClient() + version_filter = VersionFilter(git_client) + + result = version_filter.get_actual_major_redis_versions(8) + + # Should get filtered results + version_strings = [str(v[0]) for v in result] + commits = [v[1] for v in result] + expected_versions = ["8.2.1"] # Only 8.2.1 after all filtering + expected_commits = ["commit001"] + assert version_strings == expected_versions + assert commits == expected_commits \ No newline at end of file From 3468cecda9bd5aa2bc5760babaf72faaca37a73b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 20:48:33 +0300 Subject: [PATCH 194/220] Fix slack footer, return personal target --- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 4d2ebc712..b799f4e3f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -105,7 +105,7 @@ jobs: run: | image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" - footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index c90cb76e3..fab93201b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. @@ -140,7 +140,7 @@ jobs: - name: Send Slack notification run: | workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" - footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh From f9da1d4f38211a8e1508762f58cd9aed62b50cb2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 09:51:10 +0300 Subject: [PATCH 195/220] Fix sort in eol filter, update tests --- release-automation/src/stackbrew_generator/version_filter.py | 2 +- release-automation/tests/test_version_filter.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index 13db27443..bf0590795 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -86,7 +86,7 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> filtered_versions.extend(minor_group) # Sort again after filtering - filtered_versions.sort(key=lambda x: x[0], reverse=True) + filtered_versions.sort(key=lambda x: x[0].sort_key, reverse=True) console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") return filtered_versions diff --git a/release-automation/tests/test_version_filter.py b/release-automation/tests/test_version_filter.py index 7f704da28..7ebeb556d 100644 --- a/release-automation/tests/test_version_filter.py +++ b/release-automation/tests/test_version_filter.py @@ -145,6 +145,7 @@ def test_filter_eol_versions_basic(self): "v8.2.1", "v8.2.0", "v8.1.0-eol", + "v8.1.0-zoo1", "v8.1.2", "v8.0.1", "v8.0.0" From d44b5ea96b6be8d65278d0cde789e95934aafa57 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 10:54:01 +0300 Subject: [PATCH 196/220] Change slack msg format for images list --- .github/actions/common/func.sh | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 48a974c01..3220ce25d 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -334,45 +334,45 @@ prepare_releases_list() { slack_format_docker_image_urls_message() { # Parse the image URLs from JSON array - local image_urls formatted_urls release_tag footer - image_urls=$(cat) - release_tag=$1 - footer=$2 - - # Create formatted list of image URLs - formatted_urls=$(echo "$image_urls" | jq -j '.[] | "\\n• \(.)"') - -# Create Slack message payload - cat << EOF -{ -"text": "🐳 Docker Images Published for Redis: $release_tag", -"blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "🐳 Docker Images Published for Release $release_tag" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The following Docker images have been successfully published:\n\n$formatted_urls" - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "$footer" - } - ] - } -] -} -EOF + jq --arg release_tag "$1" --arg footer "$2" ' + map( + capture("(?(?[^:]+:)(?[^-]+)-(?[a-f0-9]+)-(?[^-]+)-(?[^-]+))$") + ) + as $items + | { + text: ("🐳 Docker Images Published for Redis: " + $release_tag), + blocks: [ + { + "type": "header", + "text": { "type": "plain_text", "text": ("🐳 Docker Images Published for Release " + $release_tag) } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ( + "The following Docker images have been published to Github Container Registry:\n\n" + + ( + $items + | map( + "Distribution: *" + .distro + "* " + + "Architecture: *" + .arch + "*" + + "\n```\n" + .url + "\n```" + ) + | join("\n\n") + ) + ) + } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": $footer } + ] + } + ] + } + ' } slack_format_docker_PR_message() { From 7757470497af1d5b54b7ac66a31fc99d626c65e4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:37:55 +0300 Subject: [PATCH 197/220] Add mentions and link to changes --- .github/workflows/release_publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index fab93201b..d8720de72 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -107,12 +107,15 @@ jobs: title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR BOTHERING + PLEASE IGNORE IT AND SORRY FOR THE BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} + Compare: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + + @adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh - name: PR creation results run: | From 86b550b791dd436096f73e7854d0b546ae1d6ba9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:47:42 +0300 Subject: [PATCH 198/220] New icon, move user mentions to env --- .github/actions/common/func.sh | 2 ++ .github/workflows/release_publish.yml | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 3220ce25d..13add08be 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -340,6 +340,7 @@ slack_format_docker_image_urls_message() { ) as $items | { + icon_url: ":redis-circle:", text: ("🐳 Docker Images Published for Redis: " + $release_tag), blocks: [ { @@ -383,6 +384,7 @@ slack_format_docker_PR_message() { # Create Slack message payload cat << EOF { +"icon_url": ":redis-circle:", "text": "🐳 Docker Library PR created for Redis: $release_tag", "blocks": [ { diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index d8720de72..8a97112e9 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -16,6 +16,8 @@ env: #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images + #PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" + PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" @@ -113,9 +115,9 @@ jobs: Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} - Compare: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + Compare: ${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} - @adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh + ${{ env.PR_USER_MENTIONS }} - name: PR creation results run: | From 800e079d561667b270a051a7d853d5da2a46172c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:57:09 +0300 Subject: [PATCH 199/220] Fix icon and url --- .github/actions/common/func.sh | 4 ++-- .github/workflows/release_publish.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 13add08be..4c4837000 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -340,7 +340,7 @@ slack_format_docker_image_urls_message() { ) as $items | { - icon_url: ":redis-circle:", + icon_emoji: ":redis-circle:", text: ("🐳 Docker Images Published for Redis: " + $release_tag), blocks: [ { @@ -384,7 +384,7 @@ slack_format_docker_PR_message() { # Create Slack message payload cat << EOF { -"icon_url": ":redis-circle:", +"icon_emoji": ":redis-circle:", "text": "🐳 Docker Library PR created for Redis: $release_tag", "blocks": [ { diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 8a97112e9..293d94407 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -115,7 +115,7 @@ jobs: Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} - Compare: ${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + Compare: ${{ github.server_url }}/${{ github.repository }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} ${{ env.PR_USER_MENTIONS }} From eedc99d994763500b8763904aad5eedd991d42c7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 13:47:59 +0300 Subject: [PATCH 200/220] Test failure --- .github/actions/common/func.sh | 45 +++++++++++++++++++++++++++ .github/workflows/release_publish.yml | 12 +++++++ 2 files changed, 57 insertions(+) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 4c4837000..78f8e3885 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -413,4 +413,49 @@ slack_format_docker_PR_message() { ] } EOF +} + +slack_format_docker_PR_failed_message() { + header=$1 + workflow_url=$2 + footer=$3 + if [ -z "$header" ]; then + header=" " + fi + if [ -z "$footer" ]; then + footer=" " + fi + +# Create Slack message payload + cat << EOF +{ +"icon_emoji": ":redis-circle:", +"text": "$header", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "āŒ $header" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Workflow run: $workflow_url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" + } + ] + } +] +} +EOF } \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 293d94407..8cc2cd786 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -95,6 +95,7 @@ jobs: update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis cd official-images && git diff --color cd - + exit 1 - name: Create pull request to official-images id: create-pr @@ -150,4 +151,15 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + - name: Send Failure Slack notification + if: failure() + run: | + workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 8a2aeb0e698d7e18ee89b058e39890a521efa9f4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:01:24 +0300 Subject: [PATCH 201/220] Fix failure message failure --- .github/actions/common/func.sh | 2 +- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 78f8e3885..82c4fed0c 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -415,7 +415,7 @@ slack_format_docker_PR_message() { EOF } -slack_format_docker_PR_failed_message() { +slack_format_failure_message() { header=$1 workflow_url=$2 footer=$3 diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index b799f4e3f..f4900fb56 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -110,4 +110,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 8cc2cd786..93a9f2f2c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -151,7 +151,7 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" - name: Send Failure Slack notification if: failure() @@ -162,4 +162,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 67a485b8bbacec3c7a4939e67644d64c2879316a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:14:45 +0300 Subject: [PATCH 202/220] Remove bash stackbrew implementation --- .github/actions/common/func.sh | 288 --------------------------------- test/run-shell-func-tests.sh | 234 --------------------------- 2 files changed, 522 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 82c4fed0c..36473aa4a 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -26,171 +26,6 @@ source_helper_file() { fi } -# Lists remote release branches for a specific major version (e.g., release/8.*) -git_ls_remote_major_release_version_branches() { - local remote="$1" - local major_version="$2" - execute_command --no-std -- git ls-remote --heads "$remote" "release/$major_version.*" - if [ -z "$last_cmd_stdout" ]; then - console_output 0 red "Error: No release branches found for major_version=$major_version" - return 1 - fi - echo "$last_cmd_stdout" -} - -# Lists remote tags for a specific major version (e.g., v8.*) -git_ls_remote_tags() { - local remote="$1" - local major_version="$2" - execute_command --no-std -- git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" - if [ -z "$last_cmd_stdout" ]; then - console_output 0 red "Error: No tags found for major_version=$major_version" - return 1 - fi - echo "$last_cmd_stdout" -} - -# Filters and sorts release branches by major version in reverse version order -filter_major_release_version_branches() { - local major_version="$1" - while read -r line; do - local ref="$(echo "$line" | awk '{print $2}')" - local commit="$(echo "$line" | awk '{print $1}')" - if echo "$ref" | grep -q "release/$major_version\.[0-9][0-9]*$"; then - echo "$ref $commit" - fi - done | sort -Vr -} - -# Sorts version tags in reverse version order for a specific major version -# stdin: commit ref (git ls-remote) -# stdout: version commit (vX.X.X sha1) - sorted by version -sort_version_tags() { - local major_version="$1" - local version_tag commit ref - while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*" || :)" - if [ -z "$version_tag" ]; then - console_output 2 red "Incorrect reference format: $ref" - return 1 - fi - printf "%s %s\n" "$version_tag" "$commit" - done | sort -Vr -} - -# Filters out end-of-life (EOL) versions by skipping entire minor version series marked with -eol suffix -# stdin: version commit (vX.X.X sha1) - must be sorted by version -# stdout: version commit (vX.X.X sha1) -filter_out_eol_versions() { - local major_version="$1" - local version_tag commit - local last_minor="" skip_minor="" minors="" - local major minor patch suffix - local versions - - mapfile -t versions - for line in "${versions[@]}"; do - read -r version_tag commit < <(echo "$line") - IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") - - if [ "$minor" != "$last_minor" ] && [ -n "$last_minor" ]; then - if [ -z "$skip_minor" ]; then - printf "%s" "$minors" - else - console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" - fi - minors="" - skip_minor="" - fi - last_minor="$minor" - - printf -v minors "%s%s\n" "$minors" "$version_tag $commit" - - if echo "$suffix" | grep -qi "-eol$"; then - skip_minor="$minor" - fi - done - if [ -z "$skip_minor" ]; then - printf "%s" "$minors" - else - console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" - fi -} - -# Filters Redis versions to keep only the latest patch version (and optionally the latest milestone) for each minor version -# stdin: version commit (vX.X.X sha1) - must be sorted by version -# stdout: version commit (vX.X.X sha1) -filter_actual_major_redis_versions() { - local major_version="$1" - local last_minor="" last_is_milestone="" - local ref commit version_tag - console_output 2 gray "filter_actual_major_redis_versions" - while read -r version_tag commit; do - local major minor patch suffix is_milestone - IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") - - if [ -n "$suffix" ]; then - is_milestone=1 - else - is_milestone="" - fi - - if [ "$last_minor" = "$minor" ] && [ "$last_is_milestone" = "$is_milestone" ]; then - console_output 2 gray "Skipping $version_tag, already have minor=$last_minor is_milestone=$last_is_milestone" - continue - fi - last_minor="$minor" - last_is_milestone="$is_milestone" - - console_output 2 gray "$version_tag $commit" - echo "$version_tag $commit" - done -} - -# Gets and filters actual Redis versions (tags) from a remote repository for a major version -get_actual_major_redis_versions() { - local remote="$1" - local major_version="$2" - execute_command git_ls_remote_tags "$remote" "$major_version" \ - | execute_command sort_version_tags "$major_version" \ - | execute_command filter_out_eol_versions "$major_version" \ - | execute_command filter_actual_major_redis_versions "$major_version" -} - -# Fetches unshallow refs from a remote repository for the provided list of references -git_fetch_unshallow_refs() { - local remote="$1" - local refs_to_fetch="" - while read -r line; do - local ref="$(echo "$line" | awk '{print $1}')" - refs_to_fetch="$refs_to_fetch $ref" - done - # shellcheck disable=SC2086 - execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch -} - -# Extracts the distribution name from a Dockerfile's FROM statement (supports Alpine and Debian) -extract_distro_name_from_dockerfile() { - local base_img - base_img="$(grep -m1 -i '^from' | awk '{print $2}')" - - increase_indent_level - console_output 2 gray "Extracting distro from dockerfile" - - if echo "$base_img" | grep -q 'alpine:'; then - distro="$(echo "$base_img" | tr -d ':')" - elif echo "$base_img" | grep -q 'debian:'; then - distro="$(echo "${base_img//-slim/}" | awk -F: '{print $2}')" - else - console_output 0 red "Error: Unknown base image $base_img" - decrease_indent_level - return 1 - fi - console_output 2 gray "distro=$distro" - decrease_indent_level - echo "$distro" -} - # Splits a Redis version string into major:minor:patch:suffix components redis_version_split() { local version @@ -209,129 +44,6 @@ redis_version_split() { printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" } -# Shows a file from a specific git reference (commit/branch/tag) -git_show_file_from_ref() { - local ref=$1 - local file=$2 - execute_command git show "$ref:$file" -} - -# Generates a comma-separated list of Docker tags for a Redis version and distribution -# args: redis_version distro_names is_latest is_default -# is_latest empty for non-latest, otherwise latest -# is_default 1 for default distro, otherwise not default -generate_tags_list() { - local redis_version=$1 - local distro_names=$2 - local is_latest=$3 - local is_default=$4 - - local tags versions - - local major minor patch suffix - IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") - - local mainline_version - mainline_version="$major.$minor" - - versions=("$redis_version") - # generate mainline version tag only for GA releases, e.g 8.2 and 8.2-distro - # tags will be generated only for 8.2.1 but not for 8.2.1-m01 - if [ -z "$suffix" ]; then - versions+=("$mainline_version") - fi - if [ "$is_latest" != "" ]; then - versions+=("$major") - fi - - if [ "$is_default" = 1 ]; then - tags=("${versions[@]}") - fi - - for distro_name in $distro_names; do - for v in "${versions[@]}"; do - tags+=("$v-$distro_name") - done - done - - if [ "$is_latest" != "" ]; then - if [ "$is_default" = 1 ]; then - tags+=("latest") - fi - # shellcheck disable=SC2206 - tags+=($distro_names) - fi - # shellcheck disable=SC2001 - echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" -} - -# Generates stackbrew library content (for specific major version) -# stdin: commit redis_version distro distro_version (sha1 vX.X.X alpine alpine3.21) -generate_stackbrew_library() { - local commit redis_version distro distro_version - local is_latest="" is_latest_unset=1 is_default - - local stackbrew_content="" - - mapfile -t releases - for line in "${releases[@]}"; do - read -r commit redis_version distro distro_version < <(echo "$line") - - local major minor patch suffix - IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") - - # assigning latest to the first non milestone (empty suffix) version from top - if [ "$is_latest_unset" = 1 ]; then - if [ -z "$suffix" ]; then - is_latest="$minor" - is_latest_unset="" - fi - elif [ "$is_latest" != "$minor" ]; then - is_latest="" - fi - - if echo "$distro" | grep -q 'alpine'; then - is_default="" - distro_names="$distro $distro_version" - else - is_default=1 - distro_names="$distro_version" - fi - - local tags - tags=$(generate_tags_list "$redis_version" "$distro_names" "$is_latest" "$is_default") - printf -v stackbrew_content "%s%s\n" "$stackbrew_content" "$tags" - done - printf %s "$stackbrew_content" - console_output 2 gray "$stackbrew_content" -} - -# Prepares a list of releases with commit, Redis version, distro, and distro version information -# stdin: redis_version commit -prepare_releases_list() { - local redis_version commit - local debug_output="" version_line - while read -r redis_version commit; do - for distro in debian alpine; do - local dockerfile distro_version redis_version - dockerfile=$(git_show_file_from_ref "$commit" "$distro/Dockerfile") - console_output 3 gray "$dockerfile" - - distro_version=$(echo "$dockerfile" | extract_distro_name_from_dockerfile) - # validate version - redis_version_split "$redis_version" >/dev/null - - printf -v version_line "%s %s %s %s\n" "$commit" "$redis_version" "$distro" "$distro_version" - printf "%s" "$version_line" - printf -v debug_output "%s%s" "$debug_output" "$version_line" - done - done - console_output 2 gray "Final Releases list:" - increase_indent_level - console_output 2 gray "$debug_output" - decrease_indent_level -} - slack_format_docker_image_urls_message() { # Parse the image URLs from JSON array jq --arg release_tag "$1" --arg footer "$2" ' diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index 1f4c98c94..f772ee9e4 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -10,60 +10,6 @@ set -u init_console_output -test_get_distro_name_from_dockerfile() { - distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) - assertEquals "alpine3.22" "$distro_name" - - distro_name=$(echo 'FROM debian:bookworm-slim' | extract_distro_name_from_dockerfile) - assertEquals "bookworm" "$distro_name" - - distro_name=$(echo 'FROM debian:bookworm' | extract_distro_name_from_dockerfile) - assertEquals "bookworm" "$distro_name" -} - -test_filter_major_release_version_branches() { - local input - input=$(cat < Date: Fri, 19 Sep 2025 14:15:15 +0300 Subject: [PATCH 203/220] Test build failure notification --- .github/workflows/pre-merge.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f4900fb56..d3740d7e0 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,6 +92,7 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" + exit 1 notify-slack: runs-on: ubuntu-latest @@ -110,4 +111,19 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + notify-slack-when-failed: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && failure() }} + steps: + - name: Send Failure Slack notification + run: | + workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker Build failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From e172434fcb969519d386edd0fbf58bdbbcd4d237 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:25:38 +0300 Subject: [PATCH 204/220] Checkout code for failure notification --- .github/workflows/pre-merge.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index d3740d7e0..b0b110261 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -118,6 +118,9 @@ jobs: needs: collect-image-urls if: ${{ inputs.release_tag && failure() }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Send Failure Slack notification run: | workflow_url="https://github.com/$%7B%7B%20github.repository%20%7D%7D/actions/runs/$%7B%7B%20github.run_id%20%7D%7D" From faa5f5ba509c605b4d695fa36584a41c851f684d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:38:24 +0300 Subject: [PATCH 205/220] Fix tag in build failure message --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index b0b110261..768fe5a1f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -128,5 +128,5 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - slack_format_failure_message "Docker Build failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ + slack_format_failure_message "Docker Build failed for Redis: ${{ inputs.release_tag || 'unknown'}}" "$workflow_url" "$footer" \ | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 9593d64c0cc36351a3799a7b61cee6ff6f29f46d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:50:21 +0300 Subject: [PATCH 206/220] Return debian/Dockerfile to original state --- debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/Dockerfile b/debian/Dockerfile index ad615c384..0e7d71718 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed +ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 67d28c77a53349c2b5e544ace13ed2a0b95715d2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:44:54 +0300 Subject: [PATCH 207/220] Removed failure test --- .github/workflows/pre-merge.yml | 1 - .github/workflows/release_publish.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 768fe5a1f..9190d90ac 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,7 +92,6 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" - exit 1 notify-slack: runs-on: ubuntu-latest diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 93a9f2f2c..6f7e49e89 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -95,7 +95,6 @@ jobs: update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis cd official-images && git diff --color cd - - exit 1 - name: Create pull request to official-images id: create-pr From 7672cccf8b5d5237b05f57723715ac57824fbb26 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:10:14 +0300 Subject: [PATCH 208/220] Align with redis/docker-library-redis --- .../actions/build-and-tag-locally/action.yml | 1 - .../generate-stackbrew-library.sh | 55 ------------------- .github/workflows/pre-merge.yml | 27 +++++---- 3 files changed, 13 insertions(+), 70 deletions(-) delete mode 100755 .github/actions/create-library-pr/generate-stackbrew-library.sh diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index d1e7a67ee..4409fb294 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -1,4 +1,3 @@ - name: Build and Test inputs: diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh deleted file mode 100755 index ed1fffdcc..000000000 --- a/.github/actions/create-library-pr/generate-stackbrew-library.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -if [ -z "$VERBOSITY" ]; then - VERBOSITY=1 -fi - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/func.sh" - -source_helper_file helpers.sh -source_helper_file github_helpers.sh - -init_console_output - -MAJOR_VERSION="" -REMOTE="origin" -while [[ $# -gt 0 ]]; do - case $1 in - --major-version) - MAJOR_VERSION=$2 - shift - shift - ;; - --remote) - REMOTE=$2 - shift - shift - ;; - *) - echo "Error: Unknown option $1" - exit 1 - ;; - esac -done - -if [ -z "$MAJOR_VERSION" ]; then - echo "Error: --major-version M is required as argument" - exit 1 -fi - -set -u -redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") -echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" -echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library - diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 9190d90ac..5b1b91cb6 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -18,25 +18,24 @@ on: jobs: build-and-test: - #runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} - runs-on: ["${{ contains(matrix.platform, 'arm64') && 'ARM64' || 'X64' }}", "self-hosted"] + runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} strategy: fail-fast: false matrix: distribution: - debian - # - alpine + - alpine platform: - linux/amd64 - # - linux/i386 - # - linux/arm/v5 - # - linux/arm/v6 - # - linux/arm/v7 - # - linux/mips64le - # - linux/ppc64le - # - linux/s390x + - linux/i386 + - linux/arm/v5 + - linux/arm/v6 + - linux/arm/v7 + - linux/mips64le + - linux/ppc64le + - linux/s390x - linux/arm64 - # - linux/riscv64 + - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -59,10 +58,10 @@ jobs: with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} - registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} - registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ format('ghcr.io/{0}', github.repository) }} release_tag: ${{ inputs.release_tag }} collect-image-urls: From dc3a92dab5baf1dae666763cb5b9768233a5bb95 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:11:59 +0300 Subject: [PATCH 209/220] Gitignore for release-automation --- release-automation/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 release-automation/.gitignore diff --git a/release-automation/.gitignore b/release-automation/.gitignore new file mode 100644 index 000000000..932765aeb --- /dev/null +++ b/release-automation/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +venv From 326142f45cf9eeb1c98a0041931dd4893935bddd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:17:15 +0300 Subject: [PATCH 210/220] Removed draft warning and returned official target and user mentions --- .github/workflows/release_publish.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 6f7e49e89..f21dba5ff 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,10 +13,10 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images - #PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" + PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. @@ -108,9 +108,6 @@ jobs: commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | - THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE IT AND SORRY FOR THE BOTHERING - Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} From 03cfe48b8f183fdc0bf235cbb9ccea59bc5edc8f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:22:33 +0300 Subject: [PATCH 211/220] Comment out duplicate variables --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f21dba5ff..429dd7cd6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -14,10 +14,10 @@ on: env: TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" - PR_USER_MENTIONS: "" + #PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 8a94c07f90ba9dad648fc2249fcad29ef0dbbdd0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 19:23:12 +0300 Subject: [PATCH 212/220] Fix url for redis-oss-release-automation --- .github/actions/apply-docker-version/action.yml | 2 +- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_build_and_test.yml | 8 ++++---- .github/workflows/release_publish.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 07cf7b3ec..97dd05fb0 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -17,7 +17,7 @@ runs: - name: Checkout common functions uses: actions/checkout@v4 with: - repository: redis/redis-oss-release-automation + repository: redis-developer/redis-oss-release-automation ref: main path: redis-oss-release-automation diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 5b1b91cb6..f4959d046 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -50,7 +50,7 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b4ac6a29a..bfbdfe31a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -32,13 +32,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Validate Redis Release Archive - uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main + uses: redis-developer/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -68,7 +68,7 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false @@ -76,7 +76,7 @@ jobs: - name: Merge back to release branch id: merge-back - uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main + uses: redis-developer/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 429dd7cd6..53656054f 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -66,7 +66,7 @@ jobs: echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT - name: Create version tag - uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + uses: redis-developer/redis-oss-release-automation/.github/actions/create-tag-verified@main with: tag: v${{ steps.parse-release.outputs.release_version }} ref: ${{ steps.parse-release.outputs.release_commit_sha }} From 858051c005f9c3ad360467379484c21b7b47286f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Sep 2025 13:03:27 +0000 Subject: [PATCH 213/220] 8.2.2-int --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index e34634014..c9e87cbb1 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=e6b8c98865fc022bcbddc880022694d6b4c72dc6c0b5aaefe576afb3f993c767 RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 0e7d71718..17b214d70 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2-int.tar.gz +ENV REDIS_DOWNLOAD_SHA=e6b8c98865fc022bcbddc880022694d6b4c72dc6c0b5aaefe576afb3f993c767 RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From a6f75de2c4402027eca8e72416c5e34c40ca86d4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Tue, 30 Sep 2025 17:18:22 +0300 Subject: [PATCH 214/220] Fixed regexp in notification --- .github/actions/common/func.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 36473aa4a..ae727dcea 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -48,7 +48,7 @@ slack_format_docker_image_urls_message() { # Parse the image URLs from JSON array jq --arg release_tag "$1" --arg footer "$2" ' map( - capture("(?(?[^:]+:)(?[^-]+)-(?[a-f0-9]+)-(?[^-]+)-(?[^-]+))$") + capture("(?(?[^:]+:)(?[1-9][0-9]*[.][0-9]+[.][0-9]+(-[a-z0-9]+)*)-(?[a-f0-9]{40,})-(?[^-]+)-(?[^-]+))$") ) as $items | { From e41e201810449516a8b583e7df18de6ac064b385 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 3 Oct 2025 13:50:30 +0300 Subject: [PATCH 215/220] Do not push to official library by default --- .github/workflows/release_publish.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 53656054f..3c2f79ca5 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -11,6 +11,8 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false + pr_to_official_library: + default: false env: TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images @@ -76,7 +78,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} + repository: ${{ github.event.inputs.pr_to_official_library && env.TARGET_OFFICIAL_IMAGES_REPO || env.FORKED_OFFICIAL_IMAGES_REPO }} - name: Generate stackbrew library content env: @@ -102,7 +104,7 @@ jobs: with: token: ${{ secrets.GH_TOKEN_FOR_PR }} draft: true - push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} + push-to-fork: ${{ github.event.inputs.pr_to_official_library && env.FORKED_OFFICIAL_IMAGES_REPO || '' }} path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" @@ -158,4 +160,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ - | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" From 274af947671cc282317236bbc7ac6fff27724bf2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Oct 2025 10:56:15 +0000 Subject: [PATCH 216/220] 8.2.2 --- alpine/Dockerfile | 4 ++-- debian/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpine/Dockerfile b/alpine/Dockerfile index c9e87cbb1..4cd5997df 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ # we need setpriv package as busybox provides very limited functionality setpriv \ ; -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=e6b8c98865fc022bcbddc880022694d6b4c72dc6c0b5aaefe576afb3f993c767 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2.tar.gz +ENV REDIS_DOWNLOAD_SHA=e355378d7f97efd06321fff881efc452a9673cc27b3a6d0dfd2a45fbcc83349c RUN set -eux; \ \ apk add --no-cache --virtual .build-deps \ diff --git a/debian/Dockerfile b/debian/Dockerfile index 17b214d70..9b836fc50 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2-int.tar.gz -ENV REDIS_DOWNLOAD_SHA=e6b8c98865fc022bcbddc880022694d6b4c72dc6c0b5aaefe576afb3f993c767 +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.2.tar.gz +ENV REDIS_DOWNLOAD_SHA=e355378d7f97efd06321fff881efc452a9673cc27b3a6d0dfd2a45fbcc83349c RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 957a89e29a9196429b179e7753181944c556b107 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 3 Oct 2025 14:30:03 +0300 Subject: [PATCH 217/220] Fix publish image in pre-merge workflow --- .github/workflows/pre-merge.yml | 7 ++++++- .github/workflows/release_build_and_test.yml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f4959d046..2f363aff5 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -11,6 +11,11 @@ on: description: 'Release tag to build' required: true type: string + publish_image: + description: 'Publish Docker image to GHCR' + required: false + type: boolean + default: false outputs: docker_image_urls: description: 'Array of Docker image URLs that were published' @@ -60,7 +65,7 @@ jobs: platform: ${{ matrix.platform }} registry_username: ${{ github.actor }} registry_password: ${{ secrets.GITHUB_TOKEN }} - publish_image: ${{ vars.PUBLISH_IMAGE }} + publish_image: ${{ inputs.publish_image || vars.PUBLISH_IMAGE }} registry_repository: ${{ format('ghcr.io/{0}', github.repository) }} release_tag: ${{ inputs.release_tag }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index bfbdfe31a..41299fb8d 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -57,6 +57,7 @@ jobs: secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} + publish_image: true merge-back-to-release-branch: needs: [prepare-release, build-and-test] From 7a19589822d628a7f8a6746c7983ed70311fcab3 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 3 Oct 2025 15:12:30 +0300 Subject: [PATCH 218/220] Trigger release_automation build --- .github/workflows/build_release_automation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index a9c0ff005..ed7a4fc7f 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,6 +12,9 @@ on: required: false default: true type: boolean + push: + branches: + - release/8.2 env: REGISTRY: ghcr.io From 67243f174c6656e7fe292695feb8f399f8696097 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 3 Oct 2025 15:14:23 +0300 Subject: [PATCH 219/220] Remove release automation trigger --- .github/workflows/build_release_automation.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index ed7a4fc7f..a9c0ff005 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,9 +12,6 @@ on: required: false default: true type: boolean - push: - branches: - - release/8.2 env: REGISTRY: ghcr.io From 124e4532cf20071f257a683d3b81f6bbaa05f9d6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 3 Oct 2025 15:45:06 +0300 Subject: [PATCH 220/220] Fix pr_to_official condition --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 3c2f79ca5..c57fe9d24 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -78,7 +78,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: ${{ github.event.inputs.pr_to_official_library && env.TARGET_OFFICIAL_IMAGES_REPO || env.FORKED_OFFICIAL_IMAGES_REPO }} + repository: ${{ github.event.inputs.pr_to_official_library == 'true' && env.TARGET_OFFICIAL_IMAGES_REPO || env.FORKED_OFFICIAL_IMAGES_REPO }} - name: Generate stackbrew library content env: @@ -104,7 +104,7 @@ jobs: with: token: ${{ secrets.GH_TOKEN_FOR_PR }} draft: true - push-to-fork: ${{ github.event.inputs.pr_to_official_library && env.FORKED_OFFICIAL_IMAGES_REPO || '' }} + push-to-fork: ${{ github.event.inputs.pr_to_official_library == 'true' && env.FORKED_OFFICIAL_IMAGES_REPO || '' }} path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}"