diff --git a/.github/workflows/runtime_tests.yaml b/.github/workflows/runtime_tests.yaml index 45ee4abe..32af1fa4 100644 --- a/.github/workflows/runtime_tests.yaml +++ b/.github/workflows/runtime_tests.yaml @@ -32,6 +32,14 @@ jobs: DOCKER_BUILDKIT: 1 run: docker build -t multipy --progress=plain --build-arg PYTHON_MAJOR_VERSION=${{ matrix.python-major-version }} --build-arg PYTHON_MINOR_VERSION=${{ matrix.python-minor-version }} . + - name: Set Python Version + run: | + echo "python-version=${{ matrix.python-major-version }}.${{ matrix.python-minor-version }}" >> $GITHUB_ENV + + - name: Print Python version + run: | + echo "python version = ${{ env.python-version }}" + - name: Test run: | docker run --rm multipy bash -c "if [[ ${{ matrix.python-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy" diff --git a/Dockerfile b/Dockerfile index c0fa0e27..0d3eb722 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ SHELL ["/bin/bash", "-c"] RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ apt update && DEBIAN_FRONTEND=noninteractive apt install -yq --no-install-recommends \ build-essential \ + cmake \ ca-certificates \ ccache \ curl \ @@ -44,11 +45,15 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ software-properties-common \ python-pip \ python3-pip && \ - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor -o /usr/share/keyrings/magic-key.gpg && \ - echo "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/magic-key.gpg] https://apt.kitware.com/ubuntu/ bionic main" | tee -a /etc/apt/sources.list && \ - echo "deb http://security.ubuntu.com/ubuntu focal-security main" | tee -a /etc/apt/sources.list && \ + # wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - && \ + # apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' && \ + # echo "deb http://security.ubuntu.com/ubuntu focal-security main" >> /etc/apt/sources.list && \ + # apt-add-repository ppa:ubuntu-toolchain-r/test && \ apt update && \ - apt install -y binutils cmake && \ + apt install -y binutils && \ + # cmake && \ + # apt install -y binutils cmake gcc-8 g++-8 && \ + # update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 80 --slave /usr/bin/g++ g++ /usr/bin/g++-8 && \ rm -rf /var/lib/apt/lists/* RUN /usr/sbin/update-ccache-symlinks RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache diff --git a/multipy/runtime/CMakeLists.txt b/multipy/runtime/CMakeLists.txt index da7759a3..ec9b3da8 100644 --- a/multipy/runtime/CMakeLists.txt +++ b/multipy/runtime/CMakeLists.txt @@ -7,6 +7,8 @@ cmake_minimum_required(VERSION 3.19) project(MultipyRuntime) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + # set ABI by default to 0 option(ABI_EQUALS_1 "Set ABI value to 1, by default it is set to 0. Pytorch by default builds with ABI set to 1." OFF) @@ -87,6 +89,7 @@ add_library(torch_deploy STATIC ${DEPLOY_DIR}/path_environment.cpp ${DEPLOY_DIR}/elf_file.cpp ) +# target_compile_options(torch_deploy PUBLIC -fpic -fpie) target_link_libraries(torch_deploy PRIVATE crypt pthread dl util m z ffi lzma readline nsl ncursesw panelw) # for python builtins target_link_libraries(torch_deploy PUBLIC shm torch fmt::fmt-header-only) target_include_directories(torch_deploy PRIVATE ${CMAKE_SOURCE_DIR}/../..) diff --git a/multipy/runtime/interpreter/CMakeLists.txt b/multipy/runtime/interpreter/CMakeLists.txt index 5f79236c..a29994ea 100644 --- a/multipy/runtime/interpreter/CMakeLists.txt +++ b/multipy/runtime/interpreter/CMakeLists.txt @@ -58,6 +58,8 @@ add_custom_command( COMMAND "${CMAKE_OBJCOPY}" --weaken-symbol=_PyImport_FindSharedFuncptr libpython_multipy.a ) add_custom_target(libpython_multipy DEPENDS libpython_multipy.a) +# target_compile_options(libpython_multipy PUBLIC -fpic -fpie) +set_property(TARGET libpython_multipy PROPERTY POSITION_INDEPENDENT_CODE ON) # Build the interpreter lib, designed to be standalone and dlopened # We bake the python and torch_python binding objs into libinterpreter @@ -69,11 +71,16 @@ set(INTERPRETER_LIB_SOURCES ${LINKER_SCRIPT} ) add_library(torch_deployinterpreter SHARED ${INTERPRETER_LIB_SOURCES} ${LINKER_SCRIPT}) +# target_compile_options(torch_deployinterpreter PUBLIC -fpic -fpie) add_library(multipy_torch SHARED plugin_torch.cpp) +set_property(TARGET multipy_torch PROPERTY POSITION_INDEPENDENT_CODE ON) +# target_compile_options(multipy_torch PUBLIC -fpic -fpie) + add_dependencies(torch_deployinterpreter libpython_multipy) target_link_libraries(torch_deployinterpreter PRIVATE "-Wl,--no-as-needed -rdynamic" ${CMAKE_CURRENT_BINARY_DIR}/libpython_multipy.a) +set_property(TARGET torch_deployinterpreter PROPERTY POSITION_INDEPENDENT_CODE ON) # need to ensure headers are present before any .cpp in interpreter are compiled, # but cpp themselves don't clearly depend on cpython so there is a race otherwise @@ -81,6 +88,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") target_compile_options(torch_deployinterpreter PRIVATE -fno-gnu-unique) endif() +# target_compile_options(torch_deployinterpreter PUBLIC -fpic -fpie) + target_include_directories(torch_deployinterpreter PRIVATE ${INTERPRETER_DIR}) target_include_directories(torch_deployinterpreter BEFORE PUBLIC ${Python3_INCLUDE_DIRS}) diff --git a/multipy/runtime/test_compat.py b/multipy/runtime/test_compat.py index f6318de3..7cd1788c 100644 --- a/multipy/runtime/test_compat.py +++ b/multipy/runtime/test_compat.py @@ -10,14 +10,14 @@ class TestCompat(unittest.TestCase): - def test_torchvision(self): - import torchvision # noqa: F401 + # def test_torchvision(self): + # import torchvision # noqa: F401 - def test_torchaudio(self): - import torchaudio # noqa: F401 + # def test_torchaudio(self): + # import torchaudio # noqa: F401 - def test_pytorch3d(self): - import pytorch3d # noqa: F401 + # def test_pytorch3d(self): + # import pytorch3d # noqa: F401 def test_hf_tokenizers(self): import tokenizers # noqa: F401