From ee027d07f8ad1227fe040c10c62361112f5d4290 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 31 Jan 2024 13:36:19 +0200 Subject: [PATCH 1/7] Fix GLIBC 2.10 deprecation of sys_siglist --- .ci/{py3.docker => ubuntu20.04.docker} | 0 .ci/ubuntu22.04.docker | 81 ++++++++++++++++++++++++++ Jenkinsfile.sh | 2 +- OCTOPUSSY/src/Dispatcher.cc | 6 +- OCTOPUSSY/src/Dispatcher.h | 1 + OCTOPUSSY/src/WPInterface.cc | 6 +- OCTOPUSSY/src/WPInterface.h | 1 + 7 files changed, 94 insertions(+), 3 deletions(-) rename .ci/{py3.docker => ubuntu20.04.docker} (100%) create mode 100644 .ci/ubuntu22.04.docker diff --git a/.ci/py3.docker b/.ci/ubuntu20.04.docker similarity index 100% rename from .ci/py3.docker rename to .ci/ubuntu20.04.docker diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker new file mode 100644 index 000000000..286ebfdb8 --- /dev/null +++ b/.ci/ubuntu22.04.docker @@ -0,0 +1,81 @@ +FROM ubuntu:22.04 +ENV DEBIAN_FRONTEND noninteractive +ENV DEBIAN_PRIORITY critical + +RUN apt update +RUN apt install -y libblitz0-dev python3-dev libblas-dev liblapack-dev libqdbm-dev wcslib-dev \ + libfftw3-dev python3-numpy libcfitsio-dev libboost-all-dev libboost-system-dev cmake g++ wget gfortran \ + libncurses5-dev bison libbison-dev flex libreadline6-dev python3-pip rsync \ + casacore-data python3-casacore casacore-dev python3-virtualenv + +##################################################################### +## CASArest from source +##################################################################### +RUN cd /opt && \ + wget https://github.com/casacore/casarest/archive/v1.8.1.tar.gz && \ + tar xvf v1.8.1.tar.gz && \ + rm v1.8.1.tar.gz && \ + cd /opt/casarest-1.8.1 && \ + mkdir -p build && \ + cd /opt/casarest-1.8.1/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ && \ + make -j 16 && \ + make install && \ + rm -r /opt/casarest-1.8.1 && \ + ldconfig + + +##################################################################### +## BUILD MeqTrees from source +##################################################################### +WORKDIR /opt +RUN virtualenv venv -p python3.10 --system-site-packages +RUN . /opt/venv/bin/activate && pip install setuptools wheel pip -U +# Get MeqTrees universe python packages +RUN . /opt/venv/bin/activate && python3.10 -m pip install purr \ + owlcat \ + kittens \ + 'meqtrees-cattery>=1.7.6' \ + astro-tigger-lsm \ + # possible ABI issues if this is not pinned + python-casacore==3.4.0 \ + # possible ABI issues if this is not pinned + numpy==1.21.5 + +ADD . /code +RUN mkdir /code/build && \ + cd /code/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_PYTHON_3=ON \ + .. && \ + make -j8 && \ + make install && \ + rm -r /code/build && \ + ldconfig + +##################################################################### +## Run tests +##################################################################### + +# basic install tests +RUN . /opt/venv/bin/activate && flag-ms.py --help +RUN . /opt/venv/bin/activate && meqtree-pipeliner.py --help +RUN . /opt/venv/bin/activate && pyxis --help +RUN . /opt/venv/bin/activate && python3 -c 'from Timba import mequtils' + +# run test when built +WORKDIR /src +RUN . /opt/venv/bin/activate && python3 -m pip install nose +RUN wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \ + tar -xvf v1.7.4.3.tar.gz && \ + rm v1.7.4.3.tar.gz && \ + . /opt/venv/bin/activate && \ + python3.8 -m pip install /src/pyxis-1.7.4.3 && \ + cd /src/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \ + python3.8 -m "nose" && \ + rm -r /src/pyxis-1.7.4.3 + +WORKDIR / +ENTRYPOINT ["meqtree-pipeliner.py"] +CMD ["--help"] \ No newline at end of file diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index f8d473291..ff004f1a2 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -21,4 +21,4 @@ cd $PROJECTS_DIR/meqtrees-timba IMAGENAME="mttimbapr" # build and test -docker build -f .ci/py3.docker -t "${IMAGENAME}36:$BUILD_NUMBER" --no-cache=true . +docker build -f .ci/ubuntu20.04.docker -t "${IMAGENAME}36:$BUILD_NUMBER" --no-cache=true . diff --git a/OCTOPUSSY/src/Dispatcher.cc b/OCTOPUSSY/src/Dispatcher.cc index 3dbf533ab..4978cef32 100755 --- a/OCTOPUSSY/src/Dispatcher.cc +++ b/OCTOPUSSY/src/Dispatcher.cc @@ -61,7 +61,11 @@ void Dispatcher::signalHandler (int signum,siginfo_t *,void *) #ifdef USE_THREADS if( DebugOctopussy::getDebugContext().check(2) ) { - printf("thread %ld: received signal %d (%s)\n",(long)Thread::self().id(),signum,sys_siglist[signum]); + #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 10) + printf("thread %ld: received signal %d (%s)\n",(long)Thread::self().id(),signum,strsignal(signum)); + #else + printf("thread %ld: received signal %d (%s)\n",(long)Thread::self().id(),signum,sys_siglist[signum]); + #endif } #endif sigaddset(&raisedSignals,signum); diff --git a/OCTOPUSSY/src/Dispatcher.h b/OCTOPUSSY/src/Dispatcher.h index ec0a2312a..c7fac1eec 100755 --- a/OCTOPUSSY/src/Dispatcher.h +++ b/OCTOPUSSY/src/Dispatcher.h @@ -41,6 +41,7 @@ #include #include #include +#include #pragma aid Argv diff --git a/OCTOPUSSY/src/WPInterface.cc b/OCTOPUSSY/src/WPInterface.cc index eb11668e1..a9979c6eb 100755 --- a/OCTOPUSSY/src/WPInterface.cc +++ b/OCTOPUSSY/src/WPInterface.cc @@ -1076,7 +1076,11 @@ int WPInterface::input (int fd, int flags) //##ModelId=3C7DFD240203 int WPInterface::signal (int signum) { - dprintf(1)("unhandled signal(%s)\n",sys_siglist[signum]); + #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 10) + dprintf(1)("unhandled signal(%s)\n",strsignal(signum)); + #else + dprintf(1)("unhandled signal(%s)\n",sys_siglist[signum]); + #endif return Message::ACCEPT; } diff --git a/OCTOPUSSY/src/WPInterface.h b/OCTOPUSSY/src/WPInterface.h index a5d66d294..0f92d7292 100755 --- a/OCTOPUSSY/src/WPInterface.h +++ b/OCTOPUSSY/src/WPInterface.h @@ -37,6 +37,7 @@ #include #include #include +#include namespace Octopussy { From ad01eb262c3330b93d4443bc7e07cac741c08e4d Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 31 Jan 2024 16:01:37 +0200 Subject: [PATCH 2/7] add new makems --- .ci/ubuntu22.04.docker | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index 286ebfdb8..ebcdf2066 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -24,6 +24,23 @@ RUN cd /opt && \ rm -r /opt/casarest-1.8.1 && \ ldconfig +##################################################################### +## BUILD MAKEMS FROM SOURCE AND TEST +##################################################################### +WORKDIR /opt +RUN wget https://github.com/ska-sa/makems/archive/v1.5.5.tar.gz && \ + tar xvf v1.5.5.tar.gz && \ + rm v1.5.5.tar.gz && \ + mkdir -p /opt/makems-1.5.5/LOFAR/build/gnu_opt && \ + cd /opt/makems-1.5.5/LOFAR/build/gnu_opt && \ + cmake -DCMAKE_MODULE_PATH:PATH=/opt/makems-1.5.5/LOFAR/CMake \ + -DUSE_LOG4CPLUS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr ../.. && \ + make -j 16 && \ + make install && \ + cd /opt/makems-1.5.5/test && \ + makems WSRT_makems.cfg && \ + rm -r /opt/makems-1.5.5 ##################################################################### ## BUILD MeqTrees from source @@ -66,14 +83,14 @@ RUN . /opt/venv/bin/activate && python3 -c 'from Timba import mequtils' # run test when built WORKDIR /src -RUN . /opt/venv/bin/activate && python3 -m pip install nose +RUN . /opt/venv/bin/activate && python3 -m pip install pynose RUN wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \ tar -xvf v1.7.4.3.tar.gz && \ rm v1.7.4.3.tar.gz && \ . /opt/venv/bin/activate && \ python3.8 -m pip install /src/pyxis-1.7.4.3 && \ cd /src/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \ - python3.8 -m "nose" && \ + python3.8 -m "pynose" && \ rm -r /src/pyxis-1.7.4.3 WORKDIR / From f368fa12fea134724d32f9a3bdc7c6b84c9fa126 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 31 Jan 2024 16:03:03 +0200 Subject: [PATCH 3/7] Fix typo --- .ci/ubuntu22.04.docker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index ebcdf2066..f1a7ef586 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -88,9 +88,9 @@ RUN wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \ tar -xvf v1.7.4.3.tar.gz && \ rm v1.7.4.3.tar.gz && \ . /opt/venv/bin/activate && \ - python3.8 -m pip install /src/pyxis-1.7.4.3 && \ + python3.10 -m pip install /src/pyxis-1.7.4.3 && \ cd /src/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \ - python3.8 -m "pynose" && \ + python3.10 -m "pynose" && \ rm -r /src/pyxis-1.7.4.3 WORKDIR / From 87154df821eb6c397aff6abacc9f635af094cab9 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 31 Jan 2024 16:50:54 +0200 Subject: [PATCH 4/7] add casacore-tools dependency --- .ci/ubuntu22.04.docker | 4 ++-- Jenkinsfile.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index f1a7ef586..18a5c9d79 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -6,7 +6,7 @@ RUN apt update RUN apt install -y libblitz0-dev python3-dev libblas-dev liblapack-dev libqdbm-dev wcslib-dev \ libfftw3-dev python3-numpy libcfitsio-dev libboost-all-dev libboost-system-dev cmake g++ wget gfortran \ libncurses5-dev bison libbison-dev flex libreadline6-dev python3-pip rsync \ - casacore-data python3-casacore casacore-dev python3-virtualenv + casacore-data python3-casacore casacore-dev casacore-tools python3-virtualenv ##################################################################### ## CASArest from source @@ -90,7 +90,7 @@ RUN wget https://github.com/ska-sa/pyxis/archive/v1.7.4.3.tar.gz && \ . /opt/venv/bin/activate && \ python3.10 -m pip install /src/pyxis-1.7.4.3 && \ cd /src/pyxis-1.7.4.3/Pyxis/recipes/meqtrees-batch-test && \ - python3.10 -m "pynose" && \ + pynose && \ rm -r /src/pyxis-1.7.4.3 WORKDIR / diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index ff004f1a2..35118140c 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -21,4 +21,5 @@ cd $PROJECTS_DIR/meqtrees-timba IMAGENAME="mttimbapr" # build and test -docker build -f .ci/ubuntu20.04.docker -t "${IMAGENAME}36:$BUILD_NUMBER" --no-cache=true . +docker build -f .ci/ubuntu20.04.docker -t "${IMAGENAME}38:$BUILD_NUMBER" --no-cache=true . +docker build -f .ci/ubuntu22.04.docker -t "${IMAGENAME}310:$BUILD_NUMBER" --no-cache=true . From fc2d8dd211095675be45cfdfb9aa448c48d52152 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 6 Feb 2024 16:10:55 +0200 Subject: [PATCH 5/7] .ci/ubuntu22.04.docker --- .ci/ubuntu22.04.docker | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index 18a5c9d79..de9504f4c 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -6,7 +6,29 @@ RUN apt update RUN apt install -y libblitz0-dev python3-dev libblas-dev liblapack-dev libqdbm-dev wcslib-dev \ libfftw3-dev python3-numpy libcfitsio-dev libboost-all-dev libboost-system-dev cmake g++ wget gfortran \ libncurses5-dev bison libbison-dev flex libreadline6-dev python3-pip rsync \ - casacore-data python3-casacore casacore-dev casacore-tools python3-virtualenv + python3-virtualenv libgsl-dev + +##################################################################### +## casa data from NRAO +##################################################################### +RUN mkdir -p /usr/share/casacore/data/ +WORKDIR /usr/share/casacore/data/ +RUN rsync -avz rsync://casa-rsync.nrao.edu/casa-data . + +##################################################################### +## CASAcore from source +##################################################################### +RUN cd /opt && \ + wget https://github.com/casacore/casacore/archive/v3.5.0.tar.gz && \ + tar xvf v3.5.0.tar.gz && \ + rm v3.5.0.tar.gz && \ + mkdir casacore-3.5.0/build && \ + cd /opt/casacore-3.5.0/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPRECATED=OFF -DBUILD_PYTHON=OFF -DBUILD_PYTHON3=ON ../ && \ + make -j 16 && \ + make install && \ + rm -r /opt/casacore-3.5.0 && \ + ldconfig ##################################################################### ## CASArest from source From 3ed8dc8f90aa433fa568239b059521847814d077 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 7 Feb 2024 08:13:47 +0200 Subject: [PATCH 6/7] Bump to latest casacore and python-casacore in 22.04 test --- .ci/ubuntu22.04.docker | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index de9504f4c..ef2eb59de 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -76,10 +76,12 @@ RUN . /opt/venv/bin/activate && python3.10 -m pip install purr \ kittens \ 'meqtrees-cattery>=1.7.6' \ astro-tigger-lsm \ - # possible ABI issues if this is not pinned - python-casacore==3.4.0 \ - # possible ABI issues if this is not pinned - numpy==1.21.5 + # possible ABI issues if this is not pinned to system version \ + python-casacore==3.5.2 \ + # possible ABI issues if this is not pinned to system version \ + numpy==1.21.5 \ + --no-binary 'python-casacore' + ADD . /code RUN mkdir /code/build && \ From be471d06feab44b16f4604ae8b45a7860c1eacb3 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Wed, 7 Feb 2024 09:36:43 +0200 Subject: [PATCH 7/7] WIP: switch test to kern 9 --- .ci/ubuntu22.04.docker | 64 ++++-------------------------------------- 1 file changed, 5 insertions(+), 59 deletions(-) diff --git a/.ci/ubuntu22.04.docker b/.ci/ubuntu22.04.docker index ef2eb59de..c60d0aba3 100644 --- a/.ci/ubuntu22.04.docker +++ b/.ci/ubuntu22.04.docker @@ -2,67 +2,13 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_PRIORITY critical -RUN apt update -RUN apt install -y libblitz0-dev python3-dev libblas-dev liblapack-dev libqdbm-dev wcslib-dev \ +# TODO: needed for now until kern 9 release +RUN apt update && apt install -y software-properties-common && apt-add-repository -s ppa:kernsuite/kern-dev + +RUN apt update && apt install -y libblitz0-dev python3-dev libblas-dev liblapack-dev libqdbm-dev wcslib-dev \ libfftw3-dev python3-numpy libcfitsio-dev libboost-all-dev libboost-system-dev cmake g++ wget gfortran \ libncurses5-dev bison libbison-dev flex libreadline6-dev python3-pip rsync \ - python3-virtualenv libgsl-dev - -##################################################################### -## casa data from NRAO -##################################################################### -RUN mkdir -p /usr/share/casacore/data/ -WORKDIR /usr/share/casacore/data/ -RUN rsync -avz rsync://casa-rsync.nrao.edu/casa-data . - -##################################################################### -## CASAcore from source -##################################################################### -RUN cd /opt && \ - wget https://github.com/casacore/casacore/archive/v3.5.0.tar.gz && \ - tar xvf v3.5.0.tar.gz && \ - rm v3.5.0.tar.gz && \ - mkdir casacore-3.5.0/build && \ - cd /opt/casacore-3.5.0/build && \ - cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPRECATED=OFF -DBUILD_PYTHON=OFF -DBUILD_PYTHON3=ON ../ && \ - make -j 16 && \ - make install && \ - rm -r /opt/casacore-3.5.0 && \ - ldconfig - -##################################################################### -## CASArest from source -##################################################################### -RUN cd /opt && \ - wget https://github.com/casacore/casarest/archive/v1.8.1.tar.gz && \ - tar xvf v1.8.1.tar.gz && \ - rm v1.8.1.tar.gz && \ - cd /opt/casarest-1.8.1 && \ - mkdir -p build && \ - cd /opt/casarest-1.8.1/build && \ - cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ && \ - make -j 16 && \ - make install && \ - rm -r /opt/casarest-1.8.1 && \ - ldconfig - -##################################################################### -## BUILD MAKEMS FROM SOURCE AND TEST -##################################################################### -WORKDIR /opt -RUN wget https://github.com/ska-sa/makems/archive/v1.5.5.tar.gz && \ - tar xvf v1.5.5.tar.gz && \ - rm v1.5.5.tar.gz && \ - mkdir -p /opt/makems-1.5.5/LOFAR/build/gnu_opt && \ - cd /opt/makems-1.5.5/LOFAR/build/gnu_opt && \ - cmake -DCMAKE_MODULE_PATH:PATH=/opt/makems-1.5.5/LOFAR/CMake \ - -DUSE_LOG4CPLUS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr ../.. && \ - make -j 16 && \ - make install && \ - cd /opt/makems-1.5.5/test && \ - makems WSRT_makems.cfg && \ - rm -r /opt/makems-1.5.5 + python3-virtualenv libgsl-dev casacore-dev casarest casacore-data casacore-tools makems ##################################################################### ## BUILD MeqTrees from source