From 13aac680586299f17c8219eb3d1e2d2adbea302b Mon Sep 17 00:00:00 2001 From: dciangot Date: Wed, 25 Aug 2021 10:32:50 +0200 Subject: [PATCH] spark 3.1.2 --- docker/spark/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/spark/Dockerfile b/docker/spark/Dockerfile index e46203cb..6bc67e34 100644 --- a/docker/spark/Dockerfile +++ b/docker/spark/Dockerfile @@ -84,10 +84,10 @@ RUN mkdir /tmp/jupyter_extensions && \ chmod g+w -R /opt/conda/lib/python3.9/site-packages && \ mkdir /home/$NB_USER/SWAN_projects && \ fix-permissions /home/$NB_USER && \ - cd /usr/local/ && wget https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop3.2.tgz && \ - tar -xzvf spark-3.0.1-bin-hadoop3.2.tgz && \ - mv spark-3.0.1-bin-hadoop3.2 spark && \ - rm -f spark-3.0.1-bin-hadoop3.2.tgz + cd /usr/local/ && wget https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz && \ + tar -xzvf spark-3.1.2-bin-hadoop3.2.tgz && \ + mv spark-3.1.2-bin-hadoop3.2.tgz spark && \ + rm -f spark-3.1.2-bin-hadoop3.2.tgz ADD https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.872/aws-java-sdk-bundle-1.11.872.jar /usr/local/spark/jars/ ADD https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar /usr/local/spark/jars/