Move Validation Framework to test-infra

test

fix

test

test

test

path

fix

test

test

test

fix

validate_binary

fix

test

fix

fix

actionlint

fix

test

fix

fix

fix

fix

test

test

fix

fix

.github/scripts/validate_binaries.sh

@@ -0,0 +1,121 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
+    curl ${MATRIX_INSTALLATION} -o libtorch.zip
+    unzip libtorch.zip
+else
+
+    export PYTHON_RUN="python3"
+    if [[ ${TARGET_OS} == 'windows' ]]; then
+        export PYTHON_RUN="python"
+        # Currently xpu env need a helper script to activate
+        if [[ ${MATRIX_GPU_ARCH_TYPE} == "xpu" ]]; then
+            export PYTHON_RUN="${SCRIPT_DIR}/xpu_env_helper.bat python"
+        fi
+    fi
+
+    if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
+        conda update -y -n base -c defaults conda
+    elif [[ ${TARGET_OS} != 'linux-aarch64' ]]; then
+        # Conda pinned see issue: https://github.com/ContinuumIO/anaconda-issues/issues/13350
+        conda install -y conda=23.11.0
+    fi
+    # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
+    conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+    conda activate ${ENV_NAME}
+
+    # Remove when https://github.com/pytorch/builder/issues/1985 is fixed
+    if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then
+        pip3 install numpy --force-reinstall
+    fi
+
+    INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
+    TEST_SUFFIX=""
+
+    # force-reinstall: latest version of packages are reinstalled
+    if [[ ${USE_FORCE_REINSTALL} == 'true' ]]; then
+        INSTALLATION=${INSTALLATION/"pip3 install"/"pip3 install --force-reinstall"}
+    fi
+    # extra-index-url: extra dependencies are downloaded from pypi
+    if [[ ${USE_EXTRA_INDEX_URL} == 'true' ]]; then
+        INSTALLATION=${INSTALLATION/"--index-url"/"--extra-index-url"}
+    fi
+
+    # use-meta-cdn: use meta cdn for pypi download
+    if [[ ${USE_META_CDN} == 'true' ]]; then
+        INSTALLATION=${INSTALLATION/"download.pytorch.org"/"d3kup0pazkvub8.cloudfront.net"}
+    fi
+
+
+    if [[ ${TORCH_ONLY} == 'true' ]]; then
+        INSTALLATION=${INSTALLATION/"torchvision torchaudio"/""}
+        TEST_SUFFIX=" --package torchonly"
+    fi
+
+    # if RELESE version is passed as parameter - install speific version
+    if [[ ! -z ${RELEASE_VERSION} ]]; then
+          INSTALLATION=${INSTALLATION/"torch "/"torch==${RELEASE_VERSION} "}
+          INSTALLATION=${INSTALLATION/"-y pytorch "/"-y pytorch==${RELEASE_VERSION} "}
+          INSTALLATION=${INSTALLATION/"::pytorch "/"::pytorch==${RELEASE_VERSION} "}
+
+        if [[ ${USE_VERSION_SET} == 'true' ]]; then
+          INSTALLATION=${INSTALLATION/"torchvision "/"torchvision==${VISION_RELEASE_VERSION} "}
+          INSTALLATION=${INSTALLATION/"torchaudio "/"torchaudio==${AUDIO_RELEASE_VERSION} "}
+        fi
+    fi
+
+    export OLD_PATH=${PATH}
+    # Workaround macos-arm64 runners. Issue: https://github.com/pytorch/test-infra/issues/4342
+    if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
+        export PATH="${CONDA_PREFIX}/bin:${PATH}"
+    fi
+
+    # Make sure we remove previous installation if it exists
+    if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then
+        pip3 uninstall -y torch torchaudio torchvision
+    fi
+    eval $INSTALLATION
+
+    pushd ${PWD}/.ci/pytorch/
+
+    if [[ ${MATRIX_GPU_ARCH_VERSION} == "12.6" ]]; then
+        export DESIRED_DEVTOOLSET="cxx11-abi"
+    fi
+
+    if [[ ${TARGET_OS} == 'linux' ]]; then
+        export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
+        export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
+        source ./check_binary.sh
+    fi
+
+     # We are only interested in CUDA tests and Python 3.9-3.11. Not all requirement libraries are available for 3.12 yet.
+    if [[ ${INCLUDE_TEST_OPS:-} == 'true' &&  ${MATRIX_GPU_ARCH_TYPE} == 'cuda' && ${MATRIX_PYTHON_VERSION} != "3.13" ]]; then
+        source ${SCRIPT_DIR}/validate_test_ops.sh
+    fi
+
+    # Regular smoke test
+    ${PYTHON_RUN}  ./smoke_test/smoke_test.py ${TEST_SUFFIX}
+    # For pip install also test with numpy 2.0.0 for python 3.8 or above
+    if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then
+        pip3 install numpy==2.0.0 --force-reinstall
+        ${PYTHON_RUN}  ./smoke_test/smoke_test.py ${TEST_SUFFIX}
+    fi
+
+
+    if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
+        export PATH=${OLD_PATH}
+    fi
+
+    # Use case CUDA_VISIBLE_DEVICES: https://github.com/pytorch/pytorch/issues/128819
+    if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda' ]]; then
+        python -c "import torch;import os;print(torch.cuda.device_count(), torch.__version__);os.environ['CUDA_VISIBLE_DEVICES']='0';print(torch.empty(2, device='cuda'))"
+    fi
+
+    # this is optional step
+    if [[ ${TARGET_OS} != linux*  ]]; then
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
+    fi
+    popd
+
+fi

.github/scripts/validate_pipy.sh

@@ -0,0 +1,20 @@
+conda create -yn ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+conda activate ${ENV_NAME}_pypi
+
+TEST_SUFFIX=""
+RELEASE_SUFFIX=""
+# if RELESE version is passed as parameter - install speific version
+if [[ ! -z ${RELEASE_VERSION} ]]; then
+    RELEASE_SUFFIX="==${RELEASE_VERSION}"
+fi
+
+if [[ ${TORCH_ONLY} == 'true' ]]; then
+    TEST_SUFFIX=" --package torchonly"
+    pip3 install torch${RELEASE_SUFFIX}
+else
+    pip3 install torch${RELEASE_SUFFIX} torchvision torchaudio
+fi
+
+python ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} --runtime-error-check disabled
+conda deactivate
+conda env remove -p ${ENV_NAME}_pypi

.github/scripts/validate_poetry.sh

@@ -0,0 +1,31 @@
+
+conda create -y -n ${ENV_NAME}_poetry python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+conda activate ${ENV_NAME}_poetry
+curl -sSL https://install.python-poetry.org | python3 - --git https://github.com/python-poetry/poetry.git@master
+export PATH="/root/.local/bin:$PATH"
+
+poetry --version
+poetry new test_poetry
+cd test_poetry
+
+TEST_SUFFIX=""
+if [[ ${TORCH_ONLY} == 'true' ]]; then
+    TEST_SUFFIX=" --package torchonly"
+fi
+
+RELEASE_SUFFIX=""
+# if RELESE version is passed as parameter - install speific version
+if [[ ! -z ${RELEASE_VERSION} ]]; then
+    RELEASE_SUFFIX="@${RELEASE_VERSION}"
+fi
+
+if [[ ${TORCH_ONLY} == 'true' ]]; then
+    poetry --quiet add torch${RELEASE_SUFFIX}
+else
+    poetry --quiet add torch${RELEASE_SUFFIX} torchaudio torchvision
+fi
+
+python ../test/smoke_test/smoke_test.py ${TEST_SUFFIX} --runtime-error-check disabled
+conda deactivate
+conda env remove -p ${ENV_NAME}_poetry
+cd ..

.github/scripts/validate_test_ops.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+BRANCH=""
+if [[ ${MATRIX_CHANNEL} == "test" || ${MATRIX_CHANNEL} == "release" ]]; then
+    SHORT_VERSION=${MATRIX_STABLE_VERSION%.*}
+    BRANCH="--branch release/${SHORT_VERSION}"
+fi
+
+
+# Clone the Pytorch branch
+retry git clone ${BRANCH} --depth 1 https://github.com/pytorch/pytorch.git
+retry git submodule update --init --recursive
+pushd pytorch
+
+pip install expecttest numpy pyyaml jinja2 packaging hypothesis unittest-xml-reporting scipy
+
+# Run pytorch cuda wheels validation
+# Detect ReduceLogicKernel (ReduceOp and kernel) IMA
+python test/test_ops.py -k test_dtypes_all_cuda
+# Detect BinaryMulKernel (elementwise binary functor internal mul) IMA
+python test/test_torch.py -k test_index_reduce_reduce_prod_cuda_int32
+# Detect BinaryBitwiseOpsKernels (at::native::BitwiseAndFunctor) IMA
+python test/test_binary_ufuncs.py -k test_contig_vs_every_other___rand___cuda_int32
+# Detect MaxMinElementwiseKernel (maximum) IMA
+python test/test_schema_check.py -k test_schema_correctness_clamp_cuda_int8
+
+pushd /tmp
+# Detect StepKernel (nextafter) IMA
+python -c "import torch; print(torch.nextafter(torch.tensor([-4.5149, -5.9053, -0.9516, -2.3615,  1.5591], device='cuda:0'), torch.tensor(3.8075, device='cuda:0')))"
+# Detect BinaryGeometricKernels (atan2) IMA
+python -c "import torch; x = (torch.randn((2,1,1), dtype=torch.float, device='cuda')*5).to(torch.float32); y=(torch.randn((), dtype=torch.float, device='cuda')*5).to(torch.float32); print(torch.atan2(x,y))"
+popd

.github/scripts/xpu_env_helper.bat

@@ -0,0 +1,17 @@
+call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"
+
+set args=%1
+shift
+:start
+if [%1] == [] goto done
+set args=%args% %1
+shift
+goto start
+
+:done
+if "%args%" == "" (
+    echo Usage: xpu_env_helper.bat [command] [args]
+    echo e.g. xpu_env_helper.bat icpx --version
+)
+
+%args% || exit /b 1

.github/workflows/generate_binary_build_matrix.yml

@@ -41,8 +41,8 @@ on:
         type: string
       use-only-dl-pytorch-org:
         description: "Use only download.pytorch.org when generating wheel install command?"
-        default: "false"
-        type: string
+        default: false
+        type: boolean
       build-python-only:
         description: "Generate binary build matrix for a python only package (i.e. only one python version)"
         default: "disable"

.github/workflows/generate_docker_release_matrix.yml

@@ -15,10 +15,10 @@ on:
         description: "Test infra reference to use"
         default: "main"
         type: string
-      generate_dockerhub_images: 
+      generate_dockerhub_images:
         description: "Whether to generate Docker Hub images"
         default: "false"
-        type: string
+        type: boolean
     outputs:
       matrix:
         description: "Generated build matrix"