diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6e3797b29c21e..25d918bcc25aa 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -132,6 +132,7 @@ jobs: timeout-minutes: 60 env: ARROW_HOME: /usr/local + ARROW_AZURE: ON ARROW_DATASET: ON ARROW_FLIGHT: ON ARROW_GANDIVA: ON diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index ef00f7cf4751c..b8c792008a958 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -16,6 +16,11 @@ # under the License. aws-sdk-cpp=1.11.68 +azure-core-cpp>=1.10.3 +azure-identity-cpp>=1.6.0 +azure-storage-blobs-cpp>=12.10.0 +azure-storage-common-cpp>=12.5.0 +azure-storage-files-datalake-cpp>=12.9.0 benchmark>=1.6.0 boost-cpp>=1.68.0 brotli @@ -34,6 +39,7 @@ libutf8proc lz4-c make ninja +nodejs orc pkg-config python diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index 7a54dcc86f8fa..dff1f2224809a 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -42,6 +42,13 @@ RUN mamba install -q -y \ valgrind && \ mamba clean --all +# Ensure npm, node and azurite are on path. npm and node are required to install azurite, which will then need to +# be on the path for the tests to run. +ENV PATH=/opt/conda/envs/arrow/bin:$PATH + +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + # We want to install the GCS testbench using the same Python binary that the Conda code will use. COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts RUN /arrow/ci/scripts/install_gcs_testbench.sh default @@ -50,6 +57,7 @@ COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin ENV ARROW_ACERO=ON \ + ARROW_AZURE=ON \ ARROW_BUILD_TESTS=ON \ ARROW_DATASET=ON \ ARROW_DEPENDENCY_SOURCE=CONDA \ diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh index 1d685c51a9326..a23ea8eb1cd34 100755 --- a/ci/scripts/cpp_test.sh +++ b/ci/scripts/cpp_test.sh @@ -86,6 +86,7 @@ ctest \ --label-regex unittest \ --output-on-failure \ --parallel ${n_jobs} \ + --repeat until-pass:3 \ --timeout ${ARROW_CTEST_TIMEOUT:-300} \ "${ctest_options[@]}" \ "$@" diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh index d3c6f0e6ade89..1cd1000aa3903 100755 --- a/ci/scripts/python_sdist_test.sh +++ b/ci/scripts/python_sdist_test.sh @@ -28,6 +28,7 @@ export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja} export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug} export PYARROW_WITH_ACERO=${ARROW_ACERO:-ON} +export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF} export PYARROW_WITH_S3=${ARROW_S3:-OFF} export PYARROW_WITH_ORC=${ARROW_ORC:-OFF} export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF} diff --git a/ci/scripts/python_test.sh b/ci/scripts/python_test.sh index 20ca3300c0538..7b803518494ee 100755 --- a/ci/scripts/python_test.sh +++ b/ci/scripts/python_test.sh @@ -52,6 +52,7 @@ fi : ${PYARROW_TEST_S3:=${ARROW_S3:-ON}} export PYARROW_TEST_ACERO +export PYARROW_TEST_AZURE export PYARROW_TEST_CUDA export PYARROW_TEST_DATASET export PYARROW_TEST_FLIGHT