Migrate torch::deploy test infra to pytorch/test-infra (#274)

Summary: With this PR, I propose to migrate [runtime_nightly.yaml](https://github.com/pytorch/multipy/blob/main/.github/workflows/runtime_nightly.yaml) and [runtime_tests.yaml](https://github.com/pytorch/multipy/blob/main/.github/workflows/runtime_tests.yaml) to use [pytorch/test_infra](https://github.com/pytorch/test-infra/wiki/). For this, I introduce a new reusable workflow in [build_test_release.yaml](https://github.com/pytorch/multipy/pull/274/files#diff-e9376faa19019165b5c9299c6e79ef11820184ea067877dd574bf357b7a7e01b) which is called from both the runtime tests and nightly build pipeline. The workflow then uses `pytorch/test_infra's` [linux_job.yml](https://github.com/pytorch/test-infra/blob/main/.github/workflows/) to build, test, run examples and benchmark. Example runs: - py37: https://github.com/pytorch/multipy/actions/runs/3708138595/jobs/6285368845 - py38: https://github.com/pytorch/multipy/actions/runs/3708138595/jobs/6285368958 - py39: https://github.com/pytorch/multipy/actions/runs/3708138595/jobs/6285369062 - py310: https://github.com/pytorch/multipy/actions/runs/3708138595/jobs/6285368633 Issue: #254 Pull Request resolved: #274 Reviewed By: seemethere, PaliC Differential Revision: D42069719 Pulled By: hatala91 fbshipit-source-id: c88a7863dda808a41ab49bfe56cc7b44e77566e4
pytorch · Dec 16, 2022 · 10cc663 · 10cc663
1 parent 7dd2993
commit 10cc663
Show file tree

Hide file tree

Showing 5 changed files with 183 additions and 105 deletions.
diff --git a/.github/workflows/build_test_release.yaml b/.github/workflows/build_test_release.yaml
@@ -0,0 +1,142 @@
+name: Reusable build, test and release
+
+on:
+  workflow_call:
+    inputs:
+      python3-minor-version:
+        required: true
+        type: string
+      runner:
+        required: true
+        type: string
+      distro:
+        default: centos7
+        type: string
+      compat-tests:
+        default: false
+        type: boolean
+      release:
+        default: false
+        type: boolean
+    secrets:
+      token:
+        required: true
+
+jobs:
+  build-test:
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: ${{ inputs.runner }}
+      upload-artifact: multipy_runtime_python3.${{ inputs.python3-minor-version }}
+      repository: pytorch/multipy
+      gpu-arch-type: cuda
+      gpu-arch-version: 116
+      script: |
+        python_version=3.${{ inputs.python3-minor-version }}
+
+        echo "::group::Install runtime build dependencies"
+        xargs yum install -y <build-requirements-${{ inputs.distro }}.txt
+        echo "::endgroup::"
+
+        echo "::group::Sync git submodules"
+        git submodule update --init --recursive --jobs 0
+        echo "::endgroup::"
+
+        echo "::group::Setup virtual environment"
+        if [[ ${{ inputs.python3-minor-version }} -gt 7 ]];
+        then
+          conda install -y python=${python_version} mkl mkl-include conda-build pyyaml numpy ipython
+          conda install -y -c conda-forge libpython-static=${python_version}
+          conda install -y pytorch torchvision torchaudio cpuonly -c pytorch
+          conda clean -ya;
+        else
+          conda deactivate
+          pip install virtualenv
+          git clone https://github.com/pyenv/pyenv.git ~/.pyenv
+          export CFLAGS="-fPIC -g"
+          ~/.pyenv/bin/pyenv install --force ${python_version}
+          virtualenv -p ~/.pyenv/versions/$(~/.pyenv/bin/pyenv latest ${python_version})/bin/python3 ~/venvs/multipy
+          source ~/venvs/multipy/bin/activate
+          pip install \
+          torch torchvision torchaudio \
+          --extra-index-url https://download.pytorch.org/whl/cpu;
+        fi
+        echo "::endgroup::"
+
+        echo "::group::Install"
+        python -m pip install -e . --install-option="--cudatests"
+        echo "::endgroup::"
+
+        echo "::group::Generate examples"
+        python multipy/runtime/example/generate_examples.py
+        cd examples
+        cmake -S . -B build/ -DMULTIPY_PATH=".." && cmake --build build/ --config Release -j
+        cd -
+        echo "::endgroup::"
+
+        export PYTHONPATH=$(pwd)
+        export LIBTEST_DEPLOY_LIB=$(pwd)/multipy/runtime/build/libtest_deploy_lib.so
+        export LD_LIBRARY_PATH=/opt/conda/lib/:$LD_LIBRARY_PATH
+
+        echo "::group::Test C++"
+        multipy/runtime/build/test_deploy
+        echo "::endgroup::"
+
+        echo "::group::Test Pybind"
+        python multipy/runtime/test_pybind.py
+        echo "::endgroup::"
+
+        echo "::group::Run examples"
+        examples/build/hello_world_example
+        python3 examples/quickstart/gen_package.py
+        ./examples/build/quickstart my_package.pt
+        ./examples/build/movable_example
+        echo "::endgroup::"
+
+        echo "::group::Benchmark"
+        ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet
+        echo "::endgroup::"
+
+        if [[ ${{ inputs.python3-minor-version }} -gt 7 ]];
+        then
+          echo "::group::Compat test"
+          pip install -r compat-requirements.txt
+          multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py
+          echo "::endgroup::";
+        fi
+
+        echo "::group::Test GPU"
+        # Separating GPU tests due to issues with py37 and py38: https://github.com/pytorch/multipy/issues/239
+        pip install --upgrade --force-reinstall torch \
+        --extra-index-url https://download.pytorch.org/whl/cu116
+        multipy/runtime/build/test_deploy_gpu
+        echo "::endgroup::"
+
+        echo "::group::Create tarball"
+        cp -r multipy/runtime/build/dist/* .
+        rm -rf multipy/runtime/build
+        tar -czf multipy_runtime_python${python_version}.tar.gz multipy/
+        echo "::endgroup::"
+
+        echo "::group::Move artifact for upload"
+        mv multipy_runtime_python${python_version}.tar.gz ${RUNNER_ARTIFACT_DIR}/
+        echo "::endgroup::"
+
+  release:
+    needs: [build-test]
+    runs-on: ubuntu-latest
+    if: ${{ inputs.release }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: multipy_runtime_python3.${{ inputs.python3-minor-version }}
+          path: ./
+
+      - name: Update nightly release
+        uses: pyTooling/Actions/releaser@main
+        with:
+          tag: multipy_runtime_python3.${{ inputs.python3-minor-version }}
+          rm: true
+          token: ${{ secrets.token }}
+          files: multipy_runtime_python3.${{ inputs.python3-minor-version }}.tar.gz
diff --git a/.github/workflows/runtime_nightly.yaml b/.github/workflows/runtime_nightly.yaml
@@ -6,73 +6,16 @@ on:
   workflow_dispatch:
 
 jobs:
-  unittest:
+  build-test-release:
     strategy:
       matrix:
         python3-minor-version: [7,8,9,10]
         platform: [linux.4xlarge.nvidia.gpu]
       fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - name: Checkout MultiPy
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: Setup SSH (Click me for login details)
-        uses: ./.github/actions/setup-ssh
-        with:
-          github-secret: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Clean up previous CUDA driver installations
-        shell: bash
-        run: |
-          set -x
-          yum list installed | grep nvidia || true
-          yum list installed | grep cuda || true
-          sudo yum remove -y cuda || true
-          sudo yum remove -y cuda-drivers || true
-          sudo yum remove -y "*nvidia*" || true
-      - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
-        run: |
-          bash .github/scripts/install_nvidia_utils_linux.sh || true
-          echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}"
-
-      - name: Build
-        env:
-          DOCKER_BUILDKIT: 1
-        run: nvidia-docker build -t multipy --progress=plain --build-arg PYTHON_3_MINOR_VERSION=${{ matrix.python3-minor-version }} --build-arg BUILD_CUDA_TESTS=1 .
-
-      - name: Test
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy"
-          nvidia-docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy_gpu"
-
-
-      - name: Examples
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && examples/build/hello_world_example"
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python3 examples/quickstart/gen_package.py && ./examples/build/quickstart my_package.pt"
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./examples/build/movable_example"
-
-      - name: Benchmark
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet"
-
-      - name: Set Python Version
-        run: |
-          echo "python-version=3.${{ matrix.python3-minor-version }}" >> $GITHUB_ENV
-
-      - name: Create Tarball
-        run: |
-          docker cp $(docker run -d multipy):/opt/dist/multipy .
-          tar -czvf multipy_runtime_python${{ env.python-version }}.tar.gz multipy/
-
-      - name: Update nightly release
-        uses: pyTooling/Actions/releaser@main
-        with:
-          tag: nightly-runtime-python${{ env.python-version }}
-          rm: true
-          token: ${{ secrets.GITHUB_TOKEN }}
-          files: |
-            multipy_runtime_python${{ env.python-version }}.tar.gz
+    uses: ./.github/workflows/build_test_release.yaml
+    with:
+      python3-minor-version: ${{ matrix.python3-minor-version }}
+      runner: ${{ matrix.platform }}
+      release: true
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/runtime_tests.yaml b/.github/workflows/runtime_tests.yaml
@@ -14,43 +14,10 @@ jobs:
         python3-minor-version: [7,8,9,10]
         platform: [linux.4xlarge.nvidia.gpu]
       fail-fast: false
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - name: Checkout MultiPy
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
-        uses: pytorch/test-infra/.github/actions/setup-nvidia@main
-      - name: Setup SSH (Click me for login details)
-        uses: ./.github/actions/setup-ssh
-        with:
-          github-secret: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build
-        env:
-          DOCKER_BUILDKIT: 1
-        run: nvidia-docker build -t multipy --progress=plain --build-arg PYTHON_3_MINOR_VERSION=${{ matrix.python3-minor-version }} --build-arg BUILD_CUDA_TESTS=1 .
-
-      - name: C++ Tests
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy"
-          nvidia-docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy_gpu"
-
-      - name: Pybind Tests
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python multipy/runtime/test_pybind.py"
-
-      - name: Examples
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && examples/build/hello_world_example"
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python3 examples/quickstart/gen_package.py && ./examples/build/quickstart my_package.pt"
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./examples/build/movable_example"
-
-      - name: Benchmark
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet"
-
-      - name: Compat Tests
-        run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi"
+    uses: ./.github/workflows/build_test_release.yaml
+    with:
+      python3-minor-version: ${{ matrix.python3-minor-version }}
+      runner: ${{ matrix.platform }}
+      compat-tests: true
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/build-requirements-centos7.txt b/build-requirements-centos7.txt
@@ -0,0 +1,14 @@
+binutils
+bzip2
+bzip2-devel
+gcc
+gtest-devel
+libffi-devel
+make
+openssl-devel
+readline-devel
+sqlite
+sqlite-devel
+tk-devel
+xz-devel
+zlib-devel
diff --git a/multipy/runtime/loader.cpp b/multipy/runtime/loader.cpp
@@ -67,6 +67,18 @@
 #include <multipy/runtime/loader.h>
 #include <multipy/runtime/mem_file.h>
 
+/* The ELF version installed with CentOS 7 comes with a trailing 64. For distro
+ * independent use these are mapped. */
+
+#ifndef R_AARCH64_TLS_DTPREL
+#define R_AARCH64_TLS_DTPREL R_AARCH64_TLS_DTPREL64
+
+#endif
+#ifndef R_AARCH64_TLS_DTPMOD
+#define R_AARCH64_TLS_DTPMOD R_AARCH64_TLS_DTPMOD64
+
+#endif
+
 namespace torch {
 namespace deploy {