Dockerfile

FROM airdock/oraclejdk:1.8 as metabrainz-spark-base

# Compile and install specific version of Python
# The jdk image comes with jessie which has python 3.4 which
# is not supported anymore. We install Python 3.6 here because
# 3.7 needs a version of OpenSSL that is not available in  jessie
# Based on https://github.com/docker-library/python/blob/master/3.6/jessie/Dockerfile

# Ensure that local Python build is preferred over whatever might come with the base image
ENV PATH /usr/local/bin:$PATH

# http://bugs.python.org/issue19846
# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
ENV LANG C.UTF-8

# Runtime dependencies. This includes the core packages for all of the buildDeps listed
# below. We explicitly install them so that when we `remove --auto-remove` the dev packages,
# these packages stay installed.
RUN apt-get update \
    && apt-get install -y --no-install-recommends \
                       ca-certificates \
                       netbase \
                       git \
                       libbz2-1.0 \
                       libexpat1 \
                       libffi6 \
                       libgdbm3 \
                       liblzma5 \
                       libncursesw5 \
                       libreadline6 \
                       libsqlite3-0 \
                       libssl1.0.0 \
                       libuuid1 \
                       tcl \
                       tk \
                       zlib1g wget \
	&& rm -rf /var/lib/apt/lists/*

ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
ENV PYTHON_VERSION 3.6.9

# The list of build dependencies comes from the python-docker slim version:
# https://github.com/docker-library/python/blob/408f7b8130/3.7/stretch/slim/Dockerfile#L29
RUN set -ex \
	&& buildDeps=' \
		build-essential \
		libbz2-dev \
		libexpat1-dev \
		libffi-dev \
		libgdbm-dev \
		liblzma-dev \
		libncursesw5-dev \
		libreadline-dev \
		libsqlite3-dev \
		libssl-dev \
		tk-dev \
		tcl-dev \
		uuid-dev \
		xz-utils \
		zlib1g-dev \
	' \
	&& apt-get update \
	&& apt-get install -y $buildDeps --no-install-recommends \
    \
	&& wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
	&& wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
	&& export GNUPGHOME="$(mktemp -d)" \
	&& gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \
	&& gpg --batch --verify python.tar.xz.asc python.tar.xz \
	&& { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \
	&& rm -rf "$GNUPGHOME" python.tar.xz.asc \
	&& mkdir -p /usr/src/python \
	&& tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \
	&& rm python.tar.xz \
	\
	&& cd /usr/src/python \
	&& gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \
	&& ./configure \
		--build="$gnuArch" \
		--enable-loadable-sqlite-extensions \
		--enable-shared \
		--with-system-expat \
		--with-system-ffi \
		--without-ensurepip \
	&& make -j "$(nproc)" \
	&& make install \
	&& ldconfig \
	\
	&& find /usr/local -depth \
		\( \
			\( -type d -a \( -name test -o -name tests \) \) \
			-o \
			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
		\) -exec rm -rf '{}' + \
	&& rm -rf /usr/src/python \
	\
	&& apt-get purge -y --auto-remove $buildDeps \
	&& rm -rf /var/lib/apt/lists/* \
	\
	&& python3 --version


# make some useful symlinks that are expected to exist
RUN cd /usr/local/bin \
	&& ln -s idle3 idle \
	&& ln -s pydoc3 pydoc \
	&& ln -s python3 python \
	&& ln -s python3-config python-config

# Install pip
ENV PYTHON_PIP_VERSION 19.2.3

RUN set -ex; \
	\
	wget -O get-pip.py 'https://bootstrap.pypa.io/get-pip.py'; \
	\
	python get-pip.py \
		--disable-pip-version-check \
		--no-cache-dir \
		"pip==$PYTHON_PIP_VERSION" \
	; \
	pip --version; \
	\
	find /usr/local -depth \
		\( \
			\( -type d -a \( -name test -o -name tests \) \) \
			-o \
			\( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \
		\) -exec rm -rf '{}' +; \
	rm -f get-pip.py


RUN apt-get update \
    && apt-get install -y --no-install-recommends \
    scala \
    wget \
    net-tools \
    dnsutils \
    bsdmainutils \
    xz-utils \
    pxz \
    zip \
    && rm -rf /var/lib/apt/lists/*


ENV DOCKERIZE_VERSION v0.6.1
RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz

COPY docker/apache-download.sh /apache-download.sh
ENV SPARK_VERSION 2.4.1
ENV HADOOP_VERSION 2.7
RUN cd /usr/local && \
    /apache-download.sh spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz && \
    tar xzf spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz && \
    ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION spark

RUN mkdir /rec
WORKDIR /rec
COPY requirements.txt /rec/requirements.txt
RUN pip3 install -r requirements.txt

FROM metabrainz-spark-base as metabrainz-spark-master
CMD /usr/local/spark/sbin/start-master.sh

FROM metabrainz-spark-base as metabrainz-spark-worker
CMD dockerize -wait tcp://spark-master:7077 -timeout 9999s /usr/local/spark/sbin/start-slave.sh spark://spark-master:7077

FROM metabrainz-spark-base as metabrainz-spark-jobs
COPY . /rec

FROM metabrainz-spark-base as metabrainz-spark-dev
COPY . /rec

FROM metabrainz-spark-base as metabrainz-spark-request-consumer
COPY . /rec