diff --git a/.travis.yml b/.travis.yml index 57a793ce..04fe8439 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,45 +8,59 @@ services: # This version will be also tagged as 'latest' env: global: - - LATEST="2.12-2.3.0" + - LATEST="2.13-2.8.1" # Build recommended versions based on: http://kafka.apache.org/downloads matrix: include: - - scala: "2.10" - env: KAFKA_VERSION=0.8.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.9.0.1 - - scala: 2.11 - env: KAFKA_VERSION=0.10.2.2 - - scala: 2.11 - env: KAFKA_VERSION=0.11.0.3 - - scala: 2.11 - env: KAFKA_VERSION=1.0.2 - - scala: 2.11 - env: KAFKA_VERSION=1.1.1 - - scala: 2.12 - env: KAFKA_VERSION=2.0.1 - scala: 2.12 env: KAFKA_VERSION=2.1.1 - scala: 2.12 - env: KAFKA_VERSION=2.2.1 + env: KAFKA_VERSION=2.2.2 + - scala: 2.12 + env: KAFKA_VERSION=2.3.1 - scala: 2.12 - env: KAFKA_VERSION=2.3.0 + env: KAFKA_VERSION=2.4.1 + - scala: 2.12 + env: KAFKA_VERSION=2.5.1 + - scala: 2.13 + env: KAFKA_VERSION=2.6.3 + - scala: 2.13 + env: KAFKA_VERSION=2.7.2 + - scala: 2.13 + env: KAFKA_VERSION=2.8.1 + +# Upgrade Docker Engine so we can use buildx +before_install: + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce install: - docker --version + - docker buildx version - docker-compose --version - echo "KAFKA VERSION $KAFKA_VERSION" - echo "SCALA VERSION $TRAVIS_SCALA_VERSION" - echo "LATEST VERSION $LATEST" + - if [ -z ${DOCKER_PASSWORD+x} ]; then echo "Using unauthenticated pulls on PR"; else echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; fi - export CURRENT=${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION} - - docker build --build-arg kafka_version=$KAFKA_VERSION --build-arg scala_version=$TRAVIS_SCALA_VERSION --build-arg vcs_ref=$TRAVIS_COMMIT --build-arg build_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -t wurstmeister/kafka . + + # Prepare the environment for multi-arch builds + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --use + + # Build all of the platforms and cache the result + - bash docker_buildx + + # Using the multi-arch build cache, load the current architecture image into docker for the + # subsequent docker-compose/test stuff. + - bash docker_buildx --load - docker pull confluentinc/cp-kafkacat before_script: - - docker-compose -f test/docker-compose.yml up -d zookeeper - - docker-compose -f test/docker-compose.yml scale kafka=2 + - docker-compose -f test/docker-compose.yml up -d zookeeper kafka_1 kafka_2 script: # Shellcheck main source files @@ -56,6 +70,7 @@ script: - shellcheck -x -e SC1090 -s bash *.sh **/*.sh - ./verifyImageLabels.sh # Verify docker image's label - sleep 5 # Wait for containers to start + - docker-compose logs - docker ps -a - ./runAllTests.sh # End-to-End scenario tests diff --git a/CHANGELOG.md b/CHANGELOG.md index cd5606f6..d4ed2cf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,68 @@ Changelog Kafka features are not tied to a specific kafka-docker version (ideally all changes will be merged into all branches). Therefore, this changelog will track changes to the image by date. +09-Apr-2022 +---------- + +- Switched to openjdk:11-jre-slim for multi-arch support. +- Drop support for Kafka `2.0.1` due to JDK upgrade. + +04-Oct-2021 +---------- + +- Add support for Kafka `2.8.1` + +19-July-2021 +---------- + +- Add support for Kafka `2.7.1` + +06-Jun-2021 +---------- +- Add support for darwin arm by to azul/zulu-openjdk-alpine base image + +05-Jun-2021 +----------- + +- Dropped support for versions < 2.0.1 + +30-Dec-2020 +----------- + +- Add support for Kafka `2.7.0` + +06-Aug-2020 +----------- + +- Add support for Kafka `2.6.0` + +20-Apr-2020 +----------- + +- Add support for Kafka `2.5.0` + +16-Mar-2020 +----------- + +- Add support for Kafka `2.4.1` +- Update glibc to `2.31-r0` + +20-Dec-2019 +----------- + +- Add support for Kafka `2.2.2` +- Update glibc to 2.30-r0 + +17-Dec-2019 +----------- + +- Add support for Kafka `2.4.0` + +26-Oct-2019 +----------- + +- Add support for Kafka `2.3.1` + 28-Jun-2019 ----------- diff --git a/Dockerfile b/Dockerfile index 9ccfe7a1..49668583 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,7 @@ -FROM openjdk:8u212-jre-alpine +FROM openjdk:11-jre-slim -ARG kafka_version=2.3.0 -ARG scala_version=2.12 -ARG glibc_version=2.29-r0 +ARG kafka_version=2.8.1 +ARG scala_version=2.13 ARG vcs_ref=unspecified ARG build_date=unspecified @@ -17,24 +16,41 @@ LABEL org.label-schema.name="kafka" \ ENV KAFKA_VERSION=$kafka_version \ SCALA_VERSION=$scala_version \ - KAFKA_HOME=/opt/kafka \ - GLIBC_VERSION=$glibc_version + KAFKA_HOME=/opt/kafka ENV PATH=${PATH}:${KAFKA_HOME}/bin -COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp/ - -RUN apk add --no-cache bash curl jq docker \ - && chmod a+x /tmp/*.sh \ - && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /tmp/versions.sh /usr/bin \ - && sync && /tmp/download-kafka.sh \ - && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \ - && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \ - && ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_HOME} \ - && rm /tmp/* \ - && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \ - && apk add --no-cache --allow-untrusted glibc-${GLIBC_VERSION}.apk \ - && rm glibc-${GLIBC_VERSION}.apk +COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp2/ + +RUN set -eux ; \ + apt-get update ; \ + apt-get upgrade -y ; \ + apt-get install -y --no-install-recommends jq net-tools curl wget ; \ +### BEGIN docker for CI tests + apt-get install -y --no-install-recommends gnupg lsb-release ; \ + curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg ; \ + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null ; \ + apt-get update ; \ + apt-get install -y --no-install-recommends docker-ce-cli ; \ + apt remove -y gnupg lsb-release ; \ + apt clean ; \ + apt autoremove -y ; \ + apt -f install ; \ +### END docker for CI tests +### BEGIN other for CI tests + apt-get install -y --no-install-recommends netcat ; \ +### END other for CI tests + chmod a+x /tmp2/*.sh ; \ + mv /tmp2/start-kafka.sh /tmp2/broker-list.sh /tmp2/create-topics.sh /tmp2/versions.sh /usr/bin ; \ + sync ; \ + /tmp2/download-kafka.sh ; \ + tar xfz /tmp2/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt ; \ + rm /tmp2/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz ; \ + ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_HOME} ; \ + rm -rf /tmp2 ; \ + rm -rf /var/lib/apt/lists/* COPY overrides /opt/overrides diff --git a/README.md b/README.md index 19ff8112..ca0ce84a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Docker Stars](https://img.shields.io/docker/stars/wurstmeister/kafka.svg)](https://hub.docker.com/r/wurstmeister/kafka/) [![](https://images.microbadger.com/badges/version/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own version badge on microbadger.com") [![](https://images.microbadger.com/badges/image/wurstmeister/kafka.svg)](https://microbadger.com/images/wurstmeister/kafka "Get your own image badge on microbadger.com") -[![Build Status](https://travis-ci.org/wurstmeister/kafka-docker.svg?branch=master)](https://travis-ci.org/wurstmeister/kafka-docker) +[![Build Status](https://app.travis-ci.com/wurstmeister/kafka-docker.svg?branch=master)](https://app.travis-ci.com/wurstmeister/kafka-docker) + kafka-docker ============ @@ -14,18 +15,11 @@ The image is available directly from [Docker Hub](https://hub.docker.com/r/wurst Tags and releases ----------------- -All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are: +All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `-`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). To list all available tags: -- `2.12-2.3.0` -- `2.12-2.2.1` -- `2.12-2.1.1` -- `2.12-2.0.1` -- `2.11-1.1.1` -- `2.11-1.0.2` -- `2.11-0.11.0.3` -- `2.11-0.10.2.2` -- `2.11-0.9.0.1` -- `2.10-0.8.2.2` +``` +curl -s https://registry.hub.docker.com/v2/repositories/wurstmeister/kafka/tags\?page_size\=1024 | jq -r '.results[].name' | sort -u | egrep '\d.\d{2}-.*' +``` Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image. @@ -100,7 +94,7 @@ You can configure the advertised hostname in different ways 1. explicitly, using ```KAFKA_ADVERTISED_HOST_NAME``` 2. via a command, using ```HOSTNAME_COMMAND```, e.g. ```HOSTNAME_COMMAND: "route -n | awk '/UG[ \t]/{print $$2}'"``` -When using commands, make sure you review the "Variable Substitution" section in [https://docs.docker.com/compose/compose-file/](https://docs.docker.com/compose/compose-file/) +When using commands, make sure you review the "Variable Substitution" section in [https://docs.docker.com/compose/compose-file/](https://docs.docker.com/compose/compose-file/#variable-substitution) If ```KAFKA_ADVERTISED_HOST_NAME``` is specified, it takes precedence over ```HOSTNAME_COMMAND``` diff --git a/broker-list.sh b/broker-list.sh index 73aa8220..5c5ee2d7 100755 --- a/broker-list.sh +++ b/broker-list.sh @@ -2,4 +2,4 @@ CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') BROKERS=$(for CONTAINER in ${CONTAINERS}; do docker port "$CONTAINER" 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) -echo "${BROKERS/$'\n'/,}" +echo "${BROKERS//$'\n'/,}" diff --git a/docker-compose.yml b/docker-compose.yml index 04b82c37..b4f77b38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,12 +4,16 @@ services: image: wurstmeister/zookeeper ports: - "2181:2181" + restart: unless-stopped + kafka: build: . ports: - "9092" environment: + DOCKER_API_VERSION: 1.22 KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - /var/run/docker.sock:/var/run/docker.sock + restart: unless-stopped diff --git a/docker_buildx b/docker_buildx new file mode 100644 index 00000000..41971341 --- /dev/null +++ b/docker_buildx @@ -0,0 +1,32 @@ +#!/bin/bash -e + +# Build for amd64 and arm64 +PLATFORMS="linux/amd64,linux/arm64" + +EXTRA_BUILDX_ARGS=$@ +CACHE_LOCATION="/tmp/docker-cache" + +if [[ "${EXTRA_BUILDX_ARGS}" == *"--load"* ]]; then + # We have to load the image for the current architecture only in order to run tests, so let's + # pull FROM the multi-arch build that happened previously + PLATFORMS="linux/${TRAVIS_CPU_ARCH}" + CACHE="--cache-from=type=local,src=${CACHE_LOCATION}" +elif [[ "${EXTRA_BUILDX_ARGS}" == *"--push"* ]]; then + # Push ALL architectures FROM the multi-arch build cache that happened previously + CACHE="--cache-from=type=local,src=${CACHE_LOCATION}" +else + # This is the multi-arch build that we should cache and use later + CACHE="--cache-to=type=local,dest=${CACHE_LOCATION}" +fi + +docker buildx build \ + $CACHE \ + --platform "${PLATFORMS}" \ + --progress=plain \ + --build-arg kafka_version=$KAFKA_VERSION \ + --build-arg scala_version=$TRAVIS_SCALA_VERSION \ + --build-arg vcs_ref=$TRAVIS_COMMIT \ + --build-arg build_date=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + -t wurstmeister/kafka \ + $EXTRA_BUILDX_ARGS \ + . \ No newline at end of file diff --git a/docker_push b/docker_push index 99975bb9..1cd1990d 100755 --- a/docker_push +++ b/docker_push @@ -10,5 +10,6 @@ fi echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin TARGET="$BASE_IMAGE:$IMAGE_VERSION" -docker tag "$BASE_IMAGE" "$TARGET" -docker push "$TARGET" +bash docker_buildx \ + -t "$TARGET" \ + --push diff --git a/download-kafka.sh b/download-kafka.sh index 00bf4511..58520243 100755 --- a/download-kafka.sh +++ b/download-kafka.sh @@ -1,7 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1091 -source "/usr/bin/versions.sh" +. "/usr/bin/versions.sh" FILENAME="kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" @@ -9,10 +9,10 @@ url=$(curl --stderr /dev/null "https://www.apache.org/dyn/closer.cgi?path=/kafka%20%20#%20Test%20to%20see%20if%20the%20suggested%20mirror%20has%20this%20version,%20currently%20pre%202.1.1%20versions%20#%20do%20not%20appear%20to%20be%20actively%20mirrored.%20This%20may%20also%20be%20useful%20if%20closer.cgi%20is%20down.-if%20[[%20!%20$(curl%20-s%20-f%20-I"${url}") ]]; then +if [ ! "$(curl -f -s -r 0-1 "${url}")" ]; then echo "Mirror does not have desired version, downloading direct from Apache" url="https://archive.apache.org/dist/kafka/$%7BKAFKA_VERSION%7D/$%7BFILENAME%7D" fi echo "Downloading Kafka from $url" -wget "${url}" -O "/tmp/${FILENAME}" +wget "${url}" -O "/tmp2/${FILENAME}" diff --git a/start-kafka.sh b/start-kafka.sh index 85359118..b48280a4 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -26,7 +26,7 @@ if [[ -z "$KAFKA_ADVERTISED_PORT" && \ -z "$KAFKA_LISTENERS" && \ -z "$KAFKA_ADVERTISED_LISTENERS" && \ -S /var/run/docker.sock ]]; then - KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g') + KAFKA_ADVERTISED_PORT=$(docker port "$(hostname)" $KAFKA_PORT | sed -r 's/.*:(.*)/\1/g' | head -n1) export KAFKA_ADVERTISED_PORT fi @@ -52,7 +52,7 @@ fi if [[ -n "$HOSTNAME_COMMAND" ]]; then HOSTNAME_VALUE=$(eval "$HOSTNAME_COMMAND") - # Replace any occurences of _{HOSTNAME_COMMAND} with the value + # Replace any occurrences of _{HOSTNAME_COMMAND} with the value IFS=$'\n' for VAR in $(env); do if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{HOSTNAME_COMMAND}" ]]; then @@ -65,7 +65,7 @@ fi if [[ -n "$PORT_COMMAND" ]]; then PORT_VALUE=$(eval "$PORT_COMMAND") - # Replace any occurences of _{PORT_COMMAND} with the value + # Replace any occurrences of _{PORT_COMMAND} with the value IFS=$'\n' for VAR in $(env); do if [[ $VAR =~ ^KAFKA_ && "$VAR" =~ "_{PORT_COMMAND}" ]]; then diff --git a/test/0.10/test.create-topics.kafka.sh b/test/0.10/test.create-topics.kafka.sh index b5eb4bd4..b318512f 100755 --- a/test/0.10/test.create-topics.kafka.sh +++ b/test/0.10/test.create-topics.kafka.sh @@ -11,7 +11,7 @@ testCreateTopics() { CLEANUP[0]="" TOPICS[1]="compact-$NOW" - CLEANUP[1]="compact,compression.type=snappy" + CLEANUP[1]="compression.type=snappy,cleanup.policy=compact" KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh @@ -22,7 +22,7 @@ testCreateTopics() { echo "Validating topic '$TOPIC'" EXISTS=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --list --topic "$TOPIC") - POLICY=$(/opt/kafka/bin/kafka-configs.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --entity-type topics --entity-name "$TOPIC" --describe | awk -F'cleanup.policy=' '{print $2}') + POLICY=$(/opt/kafka/bin/kafka-topics.sh --zookeeper "$KAFKA_ZOOKEEPER_CONNECT" --describe --topic "$TOPIC" | awk -F'Configs:' '{print $2}' | xargs) RESULT="$EXISTS:$POLICY" EXPECTED="$TOPIC:${CLEANUP[i]}" diff --git a/test/Readme.md b/test/Readme.md index d6c3bc3d..cdc72c19 100644 --- a/test/Readme.md +++ b/test/Readme.md @@ -8,8 +8,7 @@ To execute ``` cd test -docker-compose up -d zookeeper -docker-compose scale kafka=2 +docker-compose up -d zookeeper kafka_1 kafka_2 ./runAllTests.sh ``` diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 08800746..42316de2 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -1,21 +1,35 @@ version: '2.1' + +x-kafka-defaults: &kafka-defaults + image: wurstmeister/kafka + ports: + - "9092" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + +x-kafka-environment-defaults: &kafka-environment-defaults + HOSTNAME_COMMAND: "echo $$(hostname)" + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + services: zookeeper: image: wurstmeister/zookeeper ports: - "2181" - kafka: - image: wurstmeister/kafka - ports: - - "9092" + kafka_1: + <<: *kafka-defaults + container_name: test_kafka_1 environment: - HOSTNAME_COMMAND: "echo $$(hostname)" - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_PORT: 9092 - BROKER_ID_COMMAND: "docker inspect --format '{{ .Name }}' $$(hostname) | awk -F_ '{ printf $$NF }'" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock + <<: *kafka-environment-defaults + KAFKA_BROKER_ID: 1 + kafka_2: + <<: *kafka-defaults + container_name: test_kafka_2 + environment: + <<: *kafka-environment-defaults + KAFKA_BROKER_ID: 2 kafkatest: image: wurstmeister/kafka @@ -35,7 +49,7 @@ services: image: confluentinc/cp-kafkacat:5.0.0 environment: - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.3.0} + - KAFKA_VERSION=${KAFKA_VERSION-2.8.1} volumes: - .:/tests working_dir: /tests diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml index b87c8660..aba78bf2 100644 --- a/test/scenarios/jmx/docker-compose.yml +++ b/test/scenarios/jmx/docker-compose.yml @@ -13,7 +13,7 @@ services: KAFKA_ADVERTISED_HOST_NAME: kafka KAFKA_ADVERTISED_PORT: 9092 KAFKA_PORT: 9092 - BROKER_ID_COMMAND: "docker inspect --format '{{ .Name }}' $$(hostname) | awk -F_ '{ printf $$NF }'" + KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099" JMX_PORT: 1099