From 1f2e3e41913f794b02058f9fb3a7ddc843f3e9b8 Mon Sep 17 00:00:00 2001 From: Bryan Valentini Date: Thu, 5 Jan 2017 00:18:10 -0500 Subject: [PATCH 01/26] Bump to Spark 2.1.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0a3eb53..7cc6dc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.0.2 +ENV SPARK_VERSION 2.1.0 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From 7b268c4cee879b7e048c70d65eacd7e622fe9320 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Tue, 13 Jun 2017 14:29:51 -0700 Subject: [PATCH 02/26] Upgrading java and spark versions --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7cc6dc0..887a014 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,8 +31,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=112 -ARG JAVA_BUILD_NUMBER=15 +ARG JAVA_UPDATE_VERSION=131 +ARG JAVA_BUILD_NUMBER=11 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.1.0 +ENV SPARK_VERSION 2.1.1 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From b6bded9a2d24bb23dbcb20c85528e681babf2c84 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Tue, 13 Jun 2017 15:15:16 -0700 Subject: [PATCH 03/26] Java url change to fix build --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 887a014..9f81503 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/d54c1d3a095b4ff2b6607d096fa80163/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From 7c24cacaa78ea065d5d110c43e7eb6232c67e169 Mon Sep 17 00:00:00 2001 From: zulk666 Date: Mon, 7 Aug 2017 21:59:10 +0200 Subject: [PATCH 04/26] Bump spark to 2.2.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f81503..f1ed47a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.1.1 +ENV SPARK_VERSION 2.2.0 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From 9c7af113c8d101e0b60d1499480095ab92d899f0 Mon Sep 17 00:00:00 2001 From: Krystian Nowak Date: Wed, 3 Jan 2018 17:06:07 +0100 Subject: [PATCH 05/26] Bump to Spark 2.2.1 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1ed47a..68a2e10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,13 +57,13 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.2.0 +ENV SPARK_VERSION 2.2.1 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" ENV PATH $PATH:${SPARK_HOME}/bin RUN curl -sL --retry 3 \ - "http://d3kbcqa49mib13.cloudfront.net/$%7BSPARK_PACKAGE%7D.tgz" \ + "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}.tgz" \ | gunzip \ | tar x -C /usr/ \ && mv /usr/$SPARK_PACKAGE $SPARK_HOME \ From 93fad87b524a697681fb921d34b0652cfd5519c4 Mon Sep 17 00:00:00 2001 From: stenote Date: Tue, 23 Jan 2018 15:40:34 +0800 Subject: [PATCH 06/26] use `echo -e` to make `\n` work use `echo -e` to make `\n` work --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d195025..941b53e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ To start `spark-shell` with your AWS credentials: To do a thing with Pyspark - echo "import pyspark\nprint(pyspark.SparkContext().parallelize(range(0, 10)).count())" > count.py + echo -e "import pyspark\n\nprint(pyspark.SparkContext().parallelize(range(0, 10)).count())" > count.py docker run --rm -it -p 4040:4040 -v $(pwd)/count.py:/count.py gettyimages/spark bin/spark-submit /count.py ## docker-compose example From 298ca32796179818a58811f593b0f23c2c01f172 Mon Sep 17 00:00:00 2001 From: Mark Fletcher Date: Thu, 1 Feb 2018 16:22:25 +0000 Subject: [PATCH 07/26] Bump to Hadoop 3.0.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68a2e10..bd8a2a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN curl -sL --retry 3 --insecure \ && rm -rf $JAVA_HOME/man # HADOOP -ENV HADOOP_VERSION 2.7.3 +ENV HADOOP_VERSION 3.0.0 ENV HADOOP_HOME /usr/hadoop-$HADOOP_VERSION ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop ENV PATH $PATH:$HADOOP_HOME/bin From 8d542ae8ee5740a174c8eb1a26d96a1a6fda0f26 Mon Sep 17 00:00:00 2001 From: Krystian Nowak Date: Tue, 27 Feb 2018 21:14:07 +0100 Subject: [PATCH 08/26] Bump to Hadoop 2.8.3 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68a2e10..ea02c9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ RUN curl -sL --retry 3 --insecure \ && rm -rf $JAVA_HOME/man # HADOOP -ENV HADOOP_VERSION 2.7.3 +ENV HADOOP_VERSION 2.8.3 ENV HADOOP_HOME /usr/hadoop-$HADOOP_VERSION ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop ENV PATH $PATH:$HADOOP_HOME/bin From a899369c30f4c56f8f1c07ef3ec28d8bdde76a78 Mon Sep 17 00:00:00 2001 From: Markus Heberling Date: Thu, 15 Mar 2018 15:31:11 +0100 Subject: [PATCH 09/26] Bump to Spark 2.3.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ea02c9b..b50e775 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.2.1 +ENV SPARK_VERSION 2.3.0 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From a60ed6c94803f89fb8e7252194ee5d3feed4de4d Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Wed, 6 Jun 2018 10:51:28 +0100 Subject: [PATCH 10/26] Upgrade to Debian 9 (stretch) --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b50e775..82e102d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:jessie +FROM debian:stretch MAINTAINER Getty Images "https://github.com/gettyimages" RUN apt-get update \ diff --git a/README.md b/README.md index d195025..dc6f593 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # spark -A `debian:jessie` based [Spark](http://spark.apache.org) container. Use it in a standalone cluster with the accompanying `docker-compose.yml`, or as a base for more complex recipes. +A `debian:stretch` based [Spark](http://spark.apache.org) container. Use it in a standalone cluster with the accompanying `docker-compose.yml`, or as a base for more complex recipes. ## docker example From b949a8b7ff5ce6f497d5ffe30be15cf2a2bd054d Mon Sep 17 00:00:00 2001 From: cjekal Date: Thu, 12 Jul 2018 21:41:23 -0700 Subject: [PATCH 11/26] Updating to Spark 2.3.1 (the latest) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fc1d90c..5079ef3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.3.0 +ENV SPARK_VERSION 2.3.1 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From 0d3ed255bcca8584450b7aea25c94ef5f7fd4157 Mon Sep 17 00:00:00 2001 From: cjekal Date: Thu, 12 Jul 2018 21:41:48 -0700 Subject: [PATCH 12/26] Updating to the latest Java version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5079ef3..27bad16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,14 +31,14 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=131 +ARG JAVA_UPDATE_VERSION=171 ARG JAVA_BUILD_NUMBER=11 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/d54c1d3a095b4ff2b6607d096fa80163/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/512cd62ec5174c3487ac17c61aaa89e8/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From dd2a245731cb856d2ae1f034dc0c5b3cb4a3a82f Mon Sep 17 00:00:00 2001 From: SergeM Date: Sun, 29 Jul 2018 23:00:12 +0200 Subject: [PATCH 13/26] removed deprecated 'spark.executor.port' from configuration --- conf/master/spark-defaults.conf | 1 - conf/worker/spark-defaults.conf | 1 - 2 files changed, 2 deletions(-) diff --git a/conf/master/spark-defaults.conf b/conf/master/spark-defaults.conf index 25ac600..90be208 100644 --- a/conf/master/spark-defaults.conf +++ b/conf/master/spark-defaults.conf @@ -6,7 +6,6 @@ spark.fileserver.port 7002 spark.broadcast.port 7003 spark.replClassServer.port 7004 spark.blockManager.port 7005 -spark.executor.port 7006 spark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory spark.port.maxRetries 4 diff --git a/conf/worker/spark-defaults.conf b/conf/worker/spark-defaults.conf index 0e70efa..7c1e38a 100644 --- a/conf/worker/spark-defaults.conf +++ b/conf/worker/spark-defaults.conf @@ -6,7 +6,6 @@ spark.fileserver.port 7012 spark.broadcast.port 7013 spark.replClassServer.port 7014 spark.blockManager.port 7015 -spark.executor.port 7016 spark.broadcast.factory=org.apache.spark.broadcast.HttpBroadcastFactory spark.port.maxRetries 4 From b717c388e26425ca5fd988ea421bcc4c0b7ec0c6 Mon Sep 17 00:00:00 2001 From: SergeM Date: Tue, 31 Jul 2018 16:56:57 +0000 Subject: [PATCH 14/26] removed deprecated ports from docker-compose.yaml --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7de9ab3..30794c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,6 @@ master: - 7003 - 7004 - 7005 - - 7006 - 7077 - 6066 ports: @@ -42,7 +41,6 @@ worker: - 7013 - 7014 - 7015 - - 7016 - 8881 ports: - 8081:8081 From 7047419b8b8f589b513d26a1b572970bece6fe27 Mon Sep 17 00:00:00 2001 From: Krystian Nowak Date: Thu, 27 Sep 2018 13:35:29 +0200 Subject: [PATCH 15/26] Upgrading Java to 8u181 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27bad16..894d6e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,14 +31,14 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=171 -ARG JAVA_BUILD_NUMBER=11 +ARG JAVA_UPDATE_VERSION=181 +ARG JAVA_BUILD_NUMBER=13 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/512cd62ec5174c3487ac17c61aaa89e8/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/96a7b8442fe848ef90c96a2fad6ed6d1/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From fe1804068ba75aaf4183ddc5a693fb1803c2beaf Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Wed, 21 Nov 2018 13:49:55 -0800 Subject: [PATCH 16/26] Revert "Upgrading Java to 8u181" This reverts commit 7047419b8b8f589b513d26a1b572970bece6fe27. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 894d6e1..27bad16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,14 +31,14 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=181 -ARG JAVA_BUILD_NUMBER=13 +ARG JAVA_UPDATE_VERSION=171 +ARG JAVA_BUILD_NUMBER=11 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/96a7b8442fe848ef90c96a2fad6ed6d1/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/512cd62ec5174c3487ac17c61aaa89e8/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From 7bb2993a6c5c9b0530acf4a3c89a63e55088470d Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Mon, 26 Nov 2018 12:54:38 -0800 Subject: [PATCH 17/26] upgrading java 8 to 192 --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27bad16..0a8479a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,14 +31,14 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=171 -ARG JAVA_BUILD_NUMBER=11 +ARG JAVA_UPDATE_VERSION=192 +ARG JAVA_BUILD_NUMBER=12 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/512cd62ec5174c3487ac17c61aaa89e8/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/750e1c8617c5452694857ad95c3ee230/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From a1aa3e0d6d31633cbca3dcac088640eec603ee09 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 31 Dec 2018 09:09:54 -0800 Subject: [PATCH 18/26] Fix client version error following https://github.com/docker/compose/issues/5103 . --- docker-compose.yml | 98 +++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 30794c2..70c65e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,49 +1,51 @@ -master: - image: gettyimages/spark - command: bin/spark-class org.apache.spark.deploy.master.Master -h master - hostname: master - environment: - MASTER: spark://master:7077 - SPARK_CONF_DIR: /conf - SPARK_PUBLIC_DNS: localhost - expose: - - 7001 - - 7002 - - 7003 - - 7004 - - 7005 - - 7077 - - 6066 - ports: - - 4040:4040 - - 6066:6066 - - 7077:7077 - - 8080:8080 - volumes: - - ./conf/master:/conf - - ./data:/tmp/data +version: "2.2" +services: + master: + image: gettyimages/spark + command: bin/spark-class org.apache.spark.deploy.master.Master -h master + hostname: master + environment: + MASTER: spark://master:7077 + SPARK_CONF_DIR: /conf + SPARK_PUBLIC_DNS: localhost + expose: + - 7001 + - 7002 + - 7003 + - 7004 + - 7005 + - 7077 + - 6066 + ports: + - 4040:4040 + - 6066:6066 + - 7077:7077 + - 8080:8080 + volumes: + - ./conf/master:/conf + - ./data:/tmp/data -worker: - image: gettyimages/spark - command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077 - hostname: worker - environment: - SPARK_CONF_DIR: /conf - SPARK_WORKER_CORES: 2 - SPARK_WORKER_MEMORY: 1g - SPARK_WORKER_PORT: 8881 - SPARK_WORKER_WEBUI_PORT: 8081 - SPARK_PUBLIC_DNS: localhost - links: - - master - expose: - - 7012 - - 7013 - - 7014 - - 7015 - - 8881 - ports: - - 8081:8081 - volumes: - - ./conf/worker:/conf - - ./data:/tmp/data + worker: + image: gettyimages/spark + command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077 + hostname: worker + environment: + SPARK_CONF_DIR: /conf + SPARK_WORKER_CORES: 2 + SPARK_WORKER_MEMORY: 1g + SPARK_WORKER_PORT: 8881 + SPARK_WORKER_WEBUI_PORT: 8081 + SPARK_PUBLIC_DNS: localhost + links: + - master + expose: + - 7012 + - 7013 + - 7014 + - 7015 + - 8881 + ports: + - 8081:8081 + volumes: + - ./conf/worker:/conf + - ./data:/tmp/data From acd33ab40075b33e2ae83d23c0a0c7a8a5453734 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Wed, 2 Jan 2019 14:11:23 -0800 Subject: [PATCH 19/26] upgrading to 2.4 spark --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0a8479a..1a15b2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,7 +57,7 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.3.1 +ENV SPARK_VERSION 2.4.0 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" From fe3843c5535dedd7f64d877a195e6e1748925df4 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Tue, 29 Jan 2019 14:23:34 -0800 Subject: [PATCH 20/26] updating java version to fix build --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a15b2a..850f8e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,14 +31,14 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=192 -ARG JAVA_BUILD_NUMBER=12 +ARG JAVA_UPDATE_VERSION=202 +ARG JAVA_BUILD_NUMBER=08 ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/750e1c8617c5452694857ad95c3ee230/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/1961070e4c9b4e26a04e7f5a083f551e/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From 702335d0e6a7600808f6b25d24ccc7e519eb046b Mon Sep 17 00:00:00 2001 From: Matthew Elder Date: Mon, 15 Apr 2019 23:58:56 -0700 Subject: [PATCH 21/26] add cloudbuild yaml --- cloudbuild.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 cloudbuild.yaml diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000..67e7e20 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,24 @@ +# https://cloud.google.com/cloud-build/docs/speeding-up-builds +# https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values +substitutions: + _IMAGE: 'gcr.io/whiteblock/spark' +timeout: '45m' +steps: +# allow these steps to fail, they try to pull cache first +- name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull $_IMAGE:$BRANCH_NAME || true' ] +# build final docker image +- name: 'gcr.io/cloud-builders/docker' + args: [ + 'build', + '-t', '$_IMAGE:$BRANCH_NAME', + '-t', '$_IMAGE:$COMMIT_SHA', + '--cache-from', '$_IMAGE:$BRANCH_NAME', + '.' + ] +# push docker image tag(s) one branch, one immutable +- name: 'gcr.io/cloud-builders/docker' + args: [ 'push', '$_IMAGE:$COMMIT_SHA' ] +- name: 'gcr.io/cloud-builders/docker' + args: [ 'push', '$_IMAGE:$BRANCH_NAME' ] From 3eb6e767830c8d3af7e4bfaa5185b9fe58c5b4d4 Mon Sep 17 00:00:00 2001 From: Matthew Elder Date: Tue, 16 Apr 2019 00:07:29 -0700 Subject: [PATCH 22/26] attempt to fix spark download link --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 850f8e3..698ddfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,13 +57,13 @@ RUN curl -sL --retry 3 \ && chown -R root:root $HADOOP_HOME # SPARK -ENV SPARK_VERSION 2.4.0 +ENV SPARK_VERSION 2.4.1 ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*" ENV PATH $PATH:${SPARK_HOME}/bin RUN curl -sL --retry 3 \ - "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}.tgz" \ + "https://archive.apache.org/dist/spark/spark-$%7BSPARK_VERSION%7D/$%7BSPARK_PACKAGE%7D.tgz" \ | gunzip \ | tar x -C /usr/ \ && mv /usr/$SPARK_PACKAGE $SPARK_HOME \ From 81f79012f442992f14b05dd0b15bb844a4d49301 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Wed, 17 Apr 2019 10:56:38 -0700 Subject: [PATCH 23/26] updating java version to have public access. --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 698ddfc..e7bbda8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,15 @@ ENV PYTHONIOENCODING UTF-8 ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA -ARG JAVA_MAJOR_VERSION=8 -ARG JAVA_UPDATE_VERSION=202 -ARG JAVA_BUILD_NUMBER=08 -ENV JAVA_HOME /usr/jdk1.${JAVA_MAJOR_VERSION}.0_${JAVA_UPDATE_VERSION} +ARG JAVA_MAJOR_VERSION=12 +ARG JAVA_UPDATE_VERSION=0.1 +ARG JAVA_BUILD_NUMBER=12 +ENV JAVA_HOME /usr/jdk-${JAVA_MAJOR_VERSION}.${JAVA_UPDATE_VERSION} ENV PATH $PATH:$JAVA_HOME/bin RUN curl -sL --retry 3 --insecure \ --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-b$%7BJAVA_BUILD_NUMBER%7D/1961070e4c9b4e26a04e7f5a083f551e/server-jre-$%7BJAVA_MAJOR_VERSION%7Du$%7BJAVA_UPDATE_VERSION%7D-linux-x64.tar.gz" \ + "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7D.$%7BJAVA_UPDATE_VERSION%7D+$%7BJAVA_BUILD_NUMBER%7D/69cfe15208a647278a19ef0990eea691/jdk-$%7BJAVA_MAJOR_VERSION%7D.$%7BJAVA_UPDATE_VERSION%7D_linux-x64_bin.tar.gz" \ | gunzip \ | tar x -C /usr/ \ && ln -s $JAVA_HOME /usr/java \ From 8a52759f7685ce57b1853e7d25583076a71ac285 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Mon, 29 Apr 2019 13:30:01 -0700 Subject: [PATCH 24/26] changing to use openjdk 8 --- Dockerfile | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index e7bbda8..926496c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,19 +30,9 @@ ENV PYTHONIOENCODING UTF-8 ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA -ARG JAVA_MAJOR_VERSION=12 -ARG JAVA_UPDATE_VERSION=0.1 -ARG JAVA_BUILD_NUMBER=12 -ENV JAVA_HOME /usr/jdk-${JAVA_MAJOR_VERSION}.${JAVA_UPDATE_VERSION} - -ENV PATH $PATH:$JAVA_HOME/bin -RUN curl -sL --retry 3 --insecure \ - --header "Cookie: oraclelicense=accept-securebackup-cookie;" \ - "http://download.oracle.com/otn-pub/java/jdk/$%7BJAVA_MAJOR_VERSION%7D.$%7BJAVA_UPDATE_VERSION%7D+$%7BJAVA_BUILD_NUMBER%7D/69cfe15208a647278a19ef0990eea691/jdk-$%7BJAVA_MAJOR_VERSION%7D.$%7BJAVA_UPDATE_VERSION%7D_linux-x64_bin.tar.gz" \ - | gunzip \ - | tar x -C /usr/ \ - && ln -s $JAVA_HOME /usr/java \ - && rm -rf $JAVA_HOME/man +RUN apt-get update \ + && apt-get install -y openjdk-8-jre \ + && apt-get clean # HADOOP ENV HADOOP_VERSION 3.0.0 From cbbaa8fcd3f78b3588b2e2599007aa2062f41e26 Mon Sep 17 00:00:00 2001 From: Bryce Ageno Date: Mon, 29 Apr 2019 13:58:49 -0700 Subject: [PATCH 25/26] adding the clean for the apt get list --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 926496c..78b0c1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,8 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1 # JAVA RUN apt-get update \ && apt-get install -y openjdk-8-jre \ - && apt-get clean + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # HADOOP ENV HADOOP_VERSION 3.0.0 From 2fe5da6867c98ad5f4fae61c887d65632bd9f71c Mon Sep 17 00:00:00 2001 From: locus social inc Date: Thu, 16 May 2019 15:09:41 -0400 Subject: [PATCH 26/26] Correct the default docker instance name in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d47d343..5679220 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ To create a simplistic standalone cluster with [docker-compose](http://docs.dock The SparkUI will be running at `http://${YOUR_DOCKER_HOST}:8080` with one worker listed. To run `pyspark`, exec into a container: - docker exec -it dockerspark_master_1 /bin/bash + docker exec -it docker-spark_master_1 /bin/bash bin/pyspark To run `SparkPi`, exec into a container: - docker exec -it dockerspark_master_1 /bin/bash + docker exec -it docker-spark_master_1 /bin/bash bin/run-example SparkPi 10 ## license