diff --git a/.ci/azure-pipelines/matrix.yml b/.ci/azure-pipelines/matrix.yml index 3a6ba0ffd7..dca4473bcb 100644 --- a/.ci/azure-pipelines/matrix.yml +++ b/.ci/azure-pipelines/matrix.yml @@ -3,13 +3,17 @@ parameters: templates: - Linux: unix.yml + Ubuntu: unix.yml + CentOS: unix.yml macOS: unix.yml Windows: windows.yml - Linux: + Ubuntu: CC: gcc-9 FC: gfortran-9 + CentOS: + CC: gcc + FC: gfortran macOS: CC: gcc-9 FC: gfortran-9 @@ -31,7 +35,7 @@ parameters: # Loop/matrix parameters. # Note that these may be overridden by the including yml (typically azure-pipelines.yml). - OS_NAMES: [Windows, Linux, macOS] + OS_NAMES: [Windows, Ubuntu, CentOS, macOS] BUILD_SYSTEMS: [Make, CMake] BUILD_TYPES: [Debug, Release] MODES: [serial, smpar, dmpar, dm_sm] @@ -60,9 +64,9 @@ jobs: - ${{ if eq(parameters.WATS_DIFF, 'true') }}: - template: wats_diff.yml parameters: - # For Windows, since there is no Make-based reference, compare against Linux. + # For Windows, since there is no Make-based reference, compare against Ubuntu. ${{ if eq(OS_NAME, 'Windows') }}: - OS_NAME_LEFT: Linux + OS_NAME_LEFT: Ubuntu ${{ if ne(OS_NAME, 'Windows') }}: OS_NAME_LEFT: ${{ OS_NAME }} OS_NAME_RIGHT: ${{ OS_NAME }} @@ -78,7 +82,7 @@ jobs: - ${{ each MODE in parameters.MODES }}: - template: wats_diff.yml parameters: - OS_NAME_LEFT: Linux + OS_NAME_LEFT: Ubuntu OS_NAME_RIGHT: macOS BUILD_SYSTEM_LEFT: Make BUILD_SYSTEM_RIGHT: Make diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml index a4b858156d..1df613b2c1 100644 --- a/.ci/azure-pipelines/release.yml +++ b/.ci/azure-pipelines/release.yml @@ -9,7 +9,7 @@ steps: displayName: Create distribution package - ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}: - - bash: bash .ci/unix/delocate.sh + - bash: $(run) .ci/unix/delocate.sh displayName: Delocate - task: ArchiveFiles@2 @@ -21,9 +21,14 @@ steps: archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).tar.xz' displayName: Create distribution package -- pwsh: | - dir $(Build.ArtifactStagingDirectory) -r | % { if ($_.Name -cne $_.Name.ToLower()) { ren $_.FullName $_.Name.ToLower() } } - displayName: Rename to lowercase +- bash: | + set -ex + cd "$(Build.ArtifactStagingDirectory)" + # CentOS packages will run on most glibc-based Linux distributions + find . -name '*CentOS*' -exec bash -c 'mv $0 ${0/CentOS/Linux}' {} \; + # Rename all files to lowercase + for file in $(ls); do mv ${file} $(echo ${file} | tr '[:upper:]' '[:lower:]'); done + displayName: Rename package files # not strictly needed - task: PublishBuildArtifacts@1 diff --git a/.ci/azure-pipelines/unix.yml b/.ci/azure-pipelines/unix.yml index ab56f37095..47345c5da7 100644 --- a/.ci/azure-pipelines/unix.yml +++ b/.ci/azure-pipelines/unix.yml @@ -8,37 +8,31 @@ jobs: timeoutInMinutes: 0 pool: - ${{ if eq(parameters.OS_NAME, 'Linux') }}: + ${{ if or(eq(parameters.OS_NAME, 'Ubuntu'), eq(parameters.OS_NAME, 'CentOS')) }}: vmImage: ubuntu-latest ${{ if eq(parameters.OS_NAME, 'macOS') }}: vmImage: macOS-10.14 - ${{ if eq(parameters.OS_NAME, 'Linux') }}: - container: - image: ubuntu:16.04 - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - variables: ${{ insert }}: ${{ parameters }} - + ${{ if eq(parameters.OS_NAME, 'Ubuntu') }}: + IMAGE: ubuntu:16.04 + run: .ci/unix/run-in-docker.sh + ${{ if eq(parameters.OS_NAME, 'CentOS') }}: + IMAGE: quay.io/pypa/manylinux2010_x86_64:latest + run: .ci/unix/run-in-docker.sh + steps: - - ${{ if eq(parameters.OS_NAME, 'Linux') }}: - # https://github.com/Microsoft/azure-pipelines-agent/issues/2043#issuecomment-489692810 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: Set up sudo - - - script: .ci/unix/dump-vm-specs.sh + - script: $(run) .ci/unix/dump-vm-specs.sh displayName: Dump VM specs - - script: printenv + - script: $(run) printenv | sort displayName: Dump environment variables - - script: .ci/unix/setup-dependencies.sh + - script: $(run) .ci/unix/setup-dependencies.sh displayName: Setup dependencies - - script: .ci/unix/install-wrf.sh + - script: $(run) .ci/unix/install-wrf.sh displayName: Install WRF - ${{ if eq(parameters.WATS_RUN, 'true') }}: diff --git a/.ci/azure-pipelines/wats_diff.yml b/.ci/azure-pipelines/wats_diff.yml index 6268a7bc66..04746a48aa 100644 --- a/.ci/azure-pipelines/wats_diff.yml +++ b/.ci/azure-pipelines/wats_diff.yml @@ -6,7 +6,7 @@ jobs: displayName: 'WATS ${{ parameters.OS_NAME_LEFT }}/${{ parameters.OS_NAME_RIGHT }} ${{ parameters.BUILD_SYSTEM_LEFT }}/${{ parameters.BUILD_SYSTEM_RIGHT }}: ${{ parameters.BUILD_TYPE }}, ${{ parameters.MODE }}' pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-latest dependsOn: - ${{ parameters.OS_NAME_LEFT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }} @@ -16,14 +16,10 @@ jobs: ${{ insert }}: ${{ parameters }} steps: - - bash: bash .ci/unix/use-conda.sh + - bash: .ci/unix/use-conda.sh displayName: Enable Conda - - bash: | - set -ex - curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz - mv wats-$WATS_BRANCH wats - conda env update -n base -f wats/environment.yml + - bash: .ci/unix/install-wats.sh displayName: Install WATS - task: DownloadBuildArtifacts@0 diff --git a/.ci/azure-pipelines/wats_plots.yml b/.ci/azure-pipelines/wats_plots.yml index 4b00e34c73..fb3baf6afc 100644 --- a/.ci/azure-pipelines/wats_plots.yml +++ b/.ci/azure-pipelines/wats_plots.yml @@ -12,7 +12,7 @@ jobs: timeoutInMinutes: 0 pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-latest dependsOn: - ${{ each OS_NAME in parameters.OS_NAMES }}: @@ -27,24 +27,20 @@ jobs: ${{ insert }}: ${{ parameters.vars }} steps: - - bash: bash .ci/unix/use-conda.sh + - bash: .ci/unix/use-conda.sh displayName: Enable Conda - - bash: | - set -ex - curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz - mv wats-$WATS_BRANCH wats - conda env update -n base -f wats/environment.yml + - bash: .ci/unix/install-wats.sh displayName: Install WATS - # Plot: Linux/Make/Debug/serial against all others + # Plot: Ubuntu/Make/Debug/serial against all others - task: DownloadBuildArtifacts@0 - displayName: Retrieve reference Linux/Make/Debug/serial WATS outputs + displayName: Retrieve reference Ubuntu/Make/Debug/serial WATS outputs inputs: downloadPath: wats_outputs downloadType: specific - itemPattern: 'wats_Linux_Make_Debug_serial/**' + itemPattern: 'wats_Ubuntu_Make_Debug_serial/**' - ${{ each OS_NAME in parameters.OS_NAMES }}: - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}: @@ -52,7 +48,7 @@ jobs: - ${{ each BUILD_SYSTEM in parameters.BUILD_SYSTEMS }}: # For Windows, since there is no Make-based build, skip that combination. # Also, skip the reference combination. - - ${{ if not(or(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make')), and(eq(OS_NAME, 'Linux'), eq(BUILD_SYSTEM, 'Make'), eq(MODE, 'serial'), eq(BUILD_TYPE, 'Debug')))) }}: + - ${{ if not(or(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make')), and(eq(OS_NAME, 'Ubuntu'), eq(BUILD_SYSTEM, 'Make'), eq(MODE, 'serial'), eq(BUILD_TYPE, 'Debug')))) }}: - task: DownloadBuildArtifacts@0 displayName: Retrieve trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs @@ -67,22 +63,22 @@ jobs: set -ex O=wats_outputs W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --filter ${{ DOMAIN }} --time-idx ${{ TIME }} --append" - $W $O/wats_Linux_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }} - displayName: 'Run WATS: compute plot stats Linux/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}' + $W $O/wats_Ubuntu_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }} + displayName: 'Run WATS: compute plot stats Ubuntu/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}' - script: | rm -rf wats_outputs/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }} displayName: Remove trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs - script: | - rm -rf wats_outputs/wats_Linux_Make_Debug_serial - displayName: Remove reference Linux/Make/Debug/serial WATS outputs + rm -rf wats_outputs/wats_Ubuntu_Make_Debug_serial + displayName: Remove reference Ubuntu/Make/Debug/serial WATS outputs - ${{ each DOMAIN in parameters.DOMAINS }}: - ${{ each TIME in parameters.TIMES }}: - script: | python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/single_ref/${{ DOMAIN }}/T${{ TIME }} - displayName: 'Run WATS: create Linux/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}' + displayName: 'Run WATS: create Ubuntu/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}' - script: | rm -rf wats_stats/ diff --git a/.ci/azure-pipelines/wats_run.yml b/.ci/azure-pipelines/wats_run.yml index 68d59d1ecf..4c58052c7e 100644 --- a/.ci/azure-pipelines/wats_run.yml +++ b/.ci/azure-pipelines/wats_run.yml @@ -2,14 +2,10 @@ # Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License. steps: -- bash: bash .ci/unix/use-conda.sh +- bash: $(run) .ci/unix/use-conda.sh displayName: Enable Conda -- bash: | - set -ex - curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz - mv wats-$WATS_BRANCH wats - conda env update -n base -f wats/environment.yml +- bash: $(run) .ci/unix/install-wats.sh displayName: Install WATS - task: UniversalPackages@0 @@ -26,21 +22,7 @@ steps: find . -type f displayName: List WPS output files -- bash: | - set -ex - if [[ $MODE == dm* ]]; then mpi_flag="--mpi"; fi - if [[ $OS_NAME == macOS ]]; then - # Work around Open MPI issue - # https://github.com/open-mpi/ompi/issues/6518 - # https://github.com/open-mpi/ompi/issues/5798 - # https://www.mail-archive.com/devel@lists.open-mpi.org/msg20760.html - export OMPI_MCA_btl=self,tcp - # Disable new shared memory component of Open MPI to work around issue - # https://github.com/open-mpi/ompi/issues/7516 - export PMIX_MCA_gds=hash - fi - if [[ $BUILD_SYSTEM == "CMake" ]]; then dir_suffix="build/install"; fi - python wats/wats/main.py run --mode $WATS_MODE --wrf-dir ./$dir_suffix --wps-dir ../WPS/$dir_suffix --wps-case-output-dir wats_wps_outputs/00 --work-dir wats_work $mpi_flag +- bash: $(run) .ci/unix/run-wats.sh displayName: Run WATS - task: PublishBuildArtifacts@1 diff --git a/.ci/azure-pipelines/windows.yml b/.ci/azure-pipelines/windows.yml index 2bf1519ff9..0b5d0c4b17 100644 --- a/.ci/azure-pipelines/windows.yml +++ b/.ci/azure-pipelines/windows.yml @@ -8,7 +8,7 @@ jobs: timeoutInMinutes: 0 pool: - vmImage: windows-2019 + vmImage: windows-latest variables: ${{ insert }}: ${{ parameters }} diff --git a/.ci/unix/delocate.sh b/.ci/unix/delocate.sh old mode 100644 new mode 100755 index 984373f1fe..f09701d590 --- a/.ci/unix/delocate.sh +++ b/.ci/unix/delocate.sh @@ -10,12 +10,37 @@ cd $SCRIPTDIR/../.. if [ "$(uname)" == "Darwin" ]; then - pip3 install delocate + pip install delocate delocate-listdeps --all --depending build/install/main delocate-path build/install/main delocate-listdeps --all --depending build/install/main +elif [ "$(lsb_release -i -s)" == "CentOS" ]; then + + # Assumes we're in the manylinux Docker image. + pys=(/opt/python/cp*) + # Use the newest Python available in the image (last item). + py=${pys[@]:(-1)}/bin/python + + root_dir=$(pwd) + tmp_dir=$(mktemp -d) + cd $tmp_dir + + echo "from setuptools import setup; setup(name='app', packages=['main'], package_data={'main': ['*.exe']})" > setup.py + ln -s $root_dir/build/install/main main + $py setup.py bdist_wheel + + # CentOS uses /usr/lib64 but some manually installed dependencies end up in /usr/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib + + auditwheel repair dist/*.whl --no-update-tags + cd wheelhouse + unzip *.whl + + # /bin/cp as cp is aliased to 'cp -i' and would ask before overwriting + /bin/cp -r main/. $root_dir/build/install/main + else - echo "Unknown OS: $(uname)" + echo "Unsupported OS: $(uname)" exit 1 fi \ No newline at end of file diff --git a/.ci/unix/dump-vm-specs.sh b/.ci/unix/dump-vm-specs.sh index bc3df01d94..34ac298148 100755 --- a/.ci/unix/dump-vm-specs.sh +++ b/.ci/unix/dump-vm-specs.sh @@ -15,7 +15,11 @@ if [ "$(uname)" == "Darwin" ]; then sudo scutil --get LocalHostName || true elif [ "$(uname)" == "Linux" ]; then if [ "$(which lsb_release)" == "" ]; then - sudo apt install -y lsb-release + if [ -f /etc/redhat-release ]; then + sudo yum install -y redhat-lsb-core + else + sudo apt install -y lsb-release + fi fi lsb_release -a free -m diff --git a/.ci/unix/install-wats.sh b/.ci/unix/install-wats.sh new file mode 100755 index 0000000000..eebb80e360 --- /dev/null +++ b/.ci/unix/install-wats.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# WRF-CMake (https://github.com/WRF-CMake/wrf). +# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License. + +set -ex + +SCRIPTDIR=$(dirname "$0") +cd $SCRIPTDIR/../.. + +curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz +mv wats-$WATS_BRANCH wats +conda env update -n base -f wats/environment.yml diff --git a/.ci/unix/install-wrf.sh b/.ci/unix/install-wrf.sh index e9cb03cf7f..a0ddd3c5e1 100755 --- a/.ci/unix/install-wrf.sh +++ b/.ci/unix/install-wrf.sh @@ -78,7 +78,7 @@ elif [ $BUILD_SYSTEM == 'Make' ]; then *) echo "Invalid: $MODE"; exit 1 ;; esac - if [ "$(lsb_release -c -s)" == "trusty" ]; then + if [ "$(lsb_release -c -s)" == "trusty" -o "$(lsb_release -i -s)" == "CentOS" ]; then export HDF5=/usr export NETCDF=/usr else diff --git a/.ci/unix/run-in-docker.sh b/.ci/unix/run-in-docker.sh new file mode 100755 index 0000000000..cebcfa6825 --- /dev/null +++ b/.ci/unix/run-in-docker.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# WRF-CMake (https://github.com/WRF-CMake/wrf). +# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License. + +set -e + +SCRIPTDIR=$(dirname "$0") +ROOTDIR=$(realpath $SCRIPTDIR/../..) +cd $ROOTDIR + +container=wrf-ci + +# Create container if it doesn't exist +if [ ! "$(docker ps -q -f name=$container)" ]; then + echo "Creating Docker container $container" + set -x + docker run --name $container -t -d -v $ROOTDIR:$ROOTDIR -w $ROOTDIR -e DOCKER=1 $IMAGE + set +x + + echo "Installing sudo inside container" + if [[ $OS_NAME == CentOS ]]; then + docker exec $container sh -c "yum install -y sudo" + else + docker exec $container sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + fi +fi + +echo "Running inside container: $@" +host_envs=$(env | cut -f1 -d= | sed 's/^/-e /' | grep -v -e PATH -e HOME) +# Use login shell so that ~/.bash_profile is read. +# use-conda.sh appends to that file to modify the PATH. +docker exec $host_envs $container bash --login -c "$@" diff --git a/.ci/unix/run-wats.sh b/.ci/unix/run-wats.sh new file mode 100755 index 0000000000..e81f83f922 --- /dev/null +++ b/.ci/unix/run-wats.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# WRF-CMake (https://github.com/WRF-CMake/wrf). +# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License. + +set -ex + +SCRIPTDIR=$(dirname "$0") +cd $SCRIPTDIR/../.. + +if [ "$(lsb_release -i -s)" == "CentOS" ]; then + # CentOS uses /usr/lib64 but some manually installed dependencies end up in /usr/lib + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib +fi + +if [[ $MODE == dm* ]]; then + mpi_flag="--mpi" +fi + +if [[ $OS_NAME == macOS ]]; then + # Work around Open MPI issue + # https://github.com/open-mpi/ompi/issues/6518 + # https://github.com/open-mpi/ompi/issues/5798 + # https://www.mail-archive.com/devel@lists.open-mpi.org/msg20760.html + export OMPI_MCA_btl=self,tcp + # Disable new shared memory component of Open MPI to work around issue + # https://github.com/open-mpi/ompi/issues/7516 + export PMIX_MCA_gds=hash +fi + +if [[ $BUILD_SYSTEM == "CMake" ]]; then + dir_suffix="build/install" +fi + +python wats/wats/main.py run \ + --mode $WATS_MODE \ + --wrf-dir ./$dir_suffix \ + --wps-dir ../WPS/$dir_suffix \ + --wps-case-output-dir wats_wps_outputs/00 \ + --work-dir wats_work \ + $mpi_flag diff --git a/.ci/unix/setup-dependencies.sh b/.ci/unix/setup-dependencies.sh index 0a11ee200c..365455462a 100755 --- a/.ci/unix/setup-dependencies.sh +++ b/.ci/unix/setup-dependencies.sh @@ -13,10 +13,11 @@ HTTP_RETRIES=3 if [ "$(uname)" == "Linux" ]; then - echo "APT::Acquire::Retries \"${HTTP_RETRIES}\";" | sudo tee /etc/apt/apt.conf.d/80-retries - - sudo apt-get update - sudo apt-get install -y software-properties-common curl cmake + if [ "$(lsb_release -i -s)" == "Ubuntu" ]; then + echo "APT::Acquire::Retries \"${HTTP_RETRIES}\";" | sudo tee /etc/apt/apt.conf.d/80-retries + sudo apt-get update + sudo apt-get install -y software-properties-common curl cmake + fi if [ "$(lsb_release -c -s)" == "trusty" ]; then # We don't use latest compiler versions for 14.04 as we would otherwise @@ -35,7 +36,7 @@ if [ "$(uname)" == "Linux" ]; then # to detect whether NetCDF v4 support is available. # Since nc-config has the same CLI, just symlink. sudo ln -sf /usr/bin/nc-config /usr/bin/nf-config - else + elif [ "$(lsb_release -i -s)" == "Ubuntu" ]; then # macOS (via Homebrew) and Windows (via MSYS2) always provide the latest # compiler versions. On Ubuntu, we need to opt-in explicitly. sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test @@ -64,10 +65,75 @@ if [ "$(uname)" == "Linux" ]; then cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTS=OFF -DCMAKE_INSTALL_PREFIX=/usr .. make -j 4 sudo make install + elif [ "$(lsb_release -i -s)" == "CentOS" ]; then + sudo yum install -y zlib-devel libpng-devel jasper-devel libjpeg-devel xz + + curl -L --retry ${HTTP_RETRIES} https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.sh -o cmake.sh + sudo bash cmake.sh --prefix=/usr --exclude-subdir --skip-license + rm cmake.sh + + SZIP_VERSION=2.1.1 + curl -L --retry ${HTTP_RETRIES} https://support.hdfgroup.org/ftp/lib-external/szip/${SZIP_VERSION}/src/szip-${SZIP_VERSION}.tar.gz | tar xz + pushd szip-${SZIP_VERSION} + ./configure --prefix=/usr + sudo make install -j$(nproc) + popd + + HDF5_VERSION=1.10.5 + curl -L --retry ${HTTP_RETRIES} https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-${HDF5_VERSION}/src/CMake-hdf5-${HDF5_VERSION}.tar.gz | tar xz + pushd CMake-hdf5-${HDF5_VERSION}/hdf5-${HDF5_VERSION} + mkdir build + cd build + cmake -DCMAKE_INSTALL_PREFIX=/usr \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SKIP_RPATH=ON \ + -DHDF5_BUILD_HL_LIB=ON \ + -DHDF5_BUILD_CPP_LIB=OFF \ + -DHDF5_BUILD_FORTRAN=ON \ + -DHDF5_BUILD_TOOLS=OFF \ + -DHDF5_BUILD_EXAMPLES=OFF \ + -DHDF5_ENABLE_DEPRECATED_SYMBOLS=ON \ + -DHDF5_ENABLE_SZIP_SUPPORT=ON \ + -DHDF5_ENABLE_Z_LIB_SUPPORT=ON \ + -LA .. + sudo make install -j$(nproc) + # for WRF-Make + sudo ln -s /usr/lib/libhdf5_hl_fortran.so /usr/lib/libhdf5hl_fortran.so && + popd + + NETCDF_C_VERSION=4.6.1 + curl -L --retry ${HTTP_RETRIES} https://github.com/Unidata/netcdf-c/archive/v${NETCDF_C_VERSION}.tar.gz | tar xz + pushd netcdf-c-${NETCDF_C_VERSION} + ./configure --prefix=/usr \ + --disable-doxygen \ + --enable-logging \ + --disable-dap \ + --disable-examples \ + --disable-testsets + sudo make install -j$(nproc) + popd + + NETCDF_FORTRAN_VERSION=4.4.4 + curl -L --retry ${HTTP_RETRIES} https://github.com/Unidata/netcdf-fortran/archive/v${NETCDF_FORTRAN_VERSION}.tar.gz | tar xz + pushd netcdf-fortran-${NETCDF_FORTRAN_VERSION} + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib + ./configure --prefix=/usr --enable-static + sudo make install -j$(nproc) + popd + else + echo "The environment is not recognised" + exit 1 fi if [ $BUILD_SYSTEM == 'Make' ]; then - sudo apt-get install -y csh m4 libhdf5-serial-dev + if [ "$(lsb_release -i -s)" == "Ubuntu" ]; then + sudo apt-get install -y csh m4 libhdf5-serial-dev + elif [ "$(lsb_release -i -s)" == "CentOS" ]; then + sudo yum install -y tcsh m4 + sudo ln -sf $(which cpp) /lib/cpp + fi fi if [[ $MODE == dm* ]]; then @@ -80,8 +146,7 @@ if [ "$(uname)" == "Linux" ]; then curl -L --retry ${HTTP_RETRIES} https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xz cd mpich-${MPICH_VERSION} ./configure --prefix=/usr - make -j 4 - sudo make install + sudo make install -j$(nproc) fi fi diff --git a/.ci/unix/use-conda.sh b/.ci/unix/use-conda.sh index 6f4df38858..db7e323386 100755 --- a/.ci/unix/use-conda.sh +++ b/.ci/unix/use-conda.sh @@ -3,11 +3,13 @@ # WRF-CMake (https://github.com/WRF-CMake/wrf). # Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License. +set -ex + # See https://docs.microsoft.com/en-gb/azure/devops/pipelines/languages/anaconda. if [ "$(uname)" == "Darwin" ]; then echo "##vso[task.prependpath]$CONDA/bin" - sudo chown -R $USER $CONDA + sudo chown -R $(id -u -n) $CONDA elif [ "$(uname)" == "Linux" ]; then if [ ! -d /usr/share/miniconda ]; then curl -L --retry 3 https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh @@ -15,7 +17,11 @@ elif [ "$(uname)" == "Linux" ]; then sudo ./miniconda.sh -b -p /usr/share/miniconda rm miniconda.sh fi - echo "##vso[task.prependpath]/usr/share/miniconda/bin" + if [[ "$DOCKER" == "1" ]]; then + echo 'export PATH=/usr/share/miniconda/bin:$PATH' >> ~/.bash_profile + else + echo "##vso[task.prependpath]/usr/share/miniconda/bin" + fi sudo chown -R $(id -u -n) /usr/share/miniconda else echo "##vso[task.prependpath]$CONDA\Scripts" diff --git a/azure-pipelines-release.yml b/azure-pipelines-release.yml index 955ec1951f..4b0b09e2e0 100644 --- a/azure-pipelines-release.yml +++ b/azure-pipelines-release.yml @@ -10,7 +10,7 @@ pr: none jobs: - template: .ci/azure-pipelines/matrix.yml parameters: - OS_NAMES: [Windows, macOS] # TODO Linux + OS_NAMES: [Windows, macOS, CentOS] BUILD_SYSTEMS: [CMake] BUILD_TYPES: [Debug, Release] MODES: [serial, dmpar]