Setup build images for Pytorch & Tensorflow with base ROCm image (#557)

* Setup build images for AMD Pytorch & Tensorflow with base ROCm image * Commit piplock for amd pytorch * Update the pipfile.lock with AMD supported torch and tf * Update the Makefile to have AMD structure corrected --------- Signed-off-by: Diamond Bryant <[email protected]> Co-authored-by: atheo89 <[email protected]> Co-authored-by: Harshad Reddy Nalla <[email protected]>
opendatahub-io · Jun 27, 2024 · 9f0a837 · 9f0a837
1 parent bbba58c
commit 9f0a837
Show file tree

Hide file tree

Showing 15 changed files with 9,095 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -267,6 +267,32 @@ rstudio-c9s-python-3.9: base-c9s-python-3.9
 cuda-rstudio-c9s-python-3.9: cuda-c9s-python-3.9
 	$(call image,$@,rstudio/c9s-python-3.9,$<)
 
+####################################### Buildchain for AMD Python 3.9 using C9S #######################################
+.PHONY: amd-c9s-python-3.9
+amd-c9s-python-3.9: base-c9s-python-3.9	
+	$(call image,$@,amd/c9s-python-3.9,$<)
+
+# We are only using c9s base image here onwards, 
+# DON'T confuse due to the ubi9 mention, it just directory name.
+.PHONY: amd-minimal-c9s-python-3.9
+amd-minimal-c9s-python-3.9: amd-c9s-python-3.9
+	$(call image,$@,jupyter/minimal/ubi9-python-3.9,$<)
+
+# Build and push cuda-jupyter-datascience-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-datascience-c9s-python-3.9
+amd-jupyter-datascience-c9s-python-3.9: amd-jupyter-minimal-ubi9-python-3.9
+	$(call image,$@,jupyter/datascience/ubi9-python-3.9,$<)
+
+# Build and push cuda-jupyter-tensorflow-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-tensorflow-c9s-python-3.9
+amd-jupyter-tensorflow-c9s-python-3.9: amd-jupyter-datascience-c9s-python-3.9
+	$(call image,$@,jupyter/amd/tensorflow/ubi9-python-3.9,$<)
+
+# Build and push jupyter-pytorch-ubi9-python-3.9 image to the registry
+.PHONY: amd-jupyter-pytorch-c9s-python-3.9
+amd-jupyter-pytorch-c9s-python-3.9: amd-jupyter-datascience-c9s-python-3.9
+	$(call image,$@,jupyter/amd/pytorch/ubi9-python-3.9,$<)
+
 ####################################### Buildchain for Anaconda Python #######################################
 
 # Build and push base-anaconda-python-3.8 image to the registry

diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile b/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile
@@ -0,0 +1,27 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+LABEL name="odh-notebook-jupyter-amd-pytorch-ubi9-python-3.9" \
+    summary="Jupyter AMD pytorch notebook image for ODH notebooks" \
+    description="Jupyter AMD pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    io.k8s.display-name="Jupyter AMD pytorch notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter AMD pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/jupyter/amd/pytorch/ubi9-python-3.9" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:jupyter-amd-pytorch-ubi9-python-3.9"
+
+# Install Python packages and Jupyterlab extensions from Pipfile.lock
+COPY Pipfile.lock ./
+
+RUN echo "Installing softwares and packages" && \
+    micropipenv install && \
+    rm -f ./Pipfile.lock && \
+    # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \
+    sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
+    # Disable announcement plugin of jupyterlab \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+# Fix permissions to support pip in Openshift environments
+RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    fix-permissions /opt/app-root -P
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile b/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile
@@ -0,0 +1,62 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://download.pytorch.org/whl/rocm6.0/"
+verify_ssl = true
+name = "pytorch"
+
+[dev-packages]
+
+[packages]
+# PyTorch packages
+tensorboard = "~=2.16.2"
+torch = {version = "~=2.3.1", index = "pytorch"}
+torchvision = {version = "~=0.18.1", index = "pytorch"} # https://pytorch.org/get-started/previous-versions/#linux-and-windows-1
+
+# Datascience and useful extensions
+boto3 = "~=1.34.50"
+kafka-python = "~=2.0.2"
+matplotlib = "~=3.8.3"
+numpy = "~=1.26.4"
+pandas = "~=2.2.0"
+plotly = "~=5.20.0"
+scikit-learn = "~=1.4.0"
+scipy = "~=1.12.0"
+skl2onnx = "~=1.16.0"
+onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
+codeflare-sdk = "~=0.15.1"
+
+# DB connectors
+pymongo = "~=4.6.2"
+psycopg = "~=3.1.18"
+pyodbc = "~=5.1.0"
+mysql-connector-python = "~=8.3.0"
+
+# JupyterLab packages
+odh-elyra = "~=3.16.5"
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-bokeh = "~=3.0.7" # Upgrade would bring in jupyterlab 4
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+jupyterlab-lsp = "~=4.2.0"
+jupyterlab-widgets = "~=3.0.10"
+jupyter-resource-usage = "~=0.7.2"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+# conflict between python-lsp-server and following packages
+# due to pycodestyle dependency, pinning this, till we get 
+# a fix from python-lsp-server 
+autopep8 = "~=2.0.4"
+flake8 = "~=7.0.0"
+
+[requires]
+python_version = "3.9"