Merge branch 'master' into master

jaycedowell · Dec 9, 2024 · 60e7a92 · 60e7a92
2 parents 78de6d4 + 1edb4e5
commit 60e7a92
Show file tree

Hide file tree

Showing 199 changed files with 15,030 additions and 3,802 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -3,7 +3,7 @@ name: "Build and Test"
 "on": [push, pull_request]
 jobs:
   pre_build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     outputs:
       should_skip: ${{ steps.skip_check.outputs.should_skip }}
     steps:
@@ -19,15 +19,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [self-hosted, ubuntu-20.04, macos-latest]
-        python-version: ['2.7', '3.6', '3.8', '3.10']
-        exclude:
+        os: [self-hosted, ubuntu-latest, macos-latest]
+        python-version: ['3.8', '3.10', '3.12']
+        include:
+          - os: ubuntu-20.04
+            python-version: '3.6'
           - os: macos-latest
-            python-version: 2.7
+            python-version: '3.6'
       fail-fast: false
     steps:
       - name: "Software Install - Ubuntu"
-        if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'self-hosted' }}
+        if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-20.04' || matrix.os == 'self-hosted' }}
         run: |
           sudo apt-get update && \
           sudo apt-get install -y \
@@ -39,6 +41,7 @@ jobs:
             exuberant-ctags \
             gfortran \
             git \
+            libhwloc-dev \
             libopenblas-dev \
             libtool \
             pkg-config \
@@ -51,31 +54,33 @@ jobs:
             ctags-exuberant \
             gawk \
             gnu-sed \
+            hwloc \
             pkg-config
-      - uses: actions/setup-python@v4.3.0
+      - uses: actions/setup-python@v5.0.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: "Software Install - Python"
         run: python -m pip install \
                setuptools \
-               numpy \
+               "numpy<2" \
                matplotlib \
                contextlib2 \
                simplejson \
                pint \
                graphviz \
                ctypesgen==1.0.2 \
+               pylint \
                coverage
       - name: "Software Install - Python, part 2"
-        if: ${{ matrix.os == 'self-hosted' && matrix.python-version != '2.7' }}
+        if: ${{ matrix.os == 'self-hosted' }}
         # Setting CPLUS_INCLUDE_PATH helps pycuda find the right
         # Python header files <pyconfig.h> to use with its embedded
         # subset of Boost.
         env:
           CPLUS_INCLUDE_PATH: "${{ env.pythonLocation }}/include/python\
             ${{ matrix.python-version }}"
         run: python -m pip install \
-               cupy-cuda112 \
+               cupy-cuda12x \
                pycuda \
                numba \
                jupyterlab \
@@ -109,14 +114,27 @@ jobs:
           coverage run --source=bifrost.ring,bifrost,bifrost.pipeline \
             -m unittest discover
           coverage xml
+      - name: "Test, part 2"
+        if: ${{ matrix.os == 'self-hosted' }}
+        env:
+          LD_LIBRARY_PATH: /usr/local/lib:${{ env.LD_LIBRARY_PATH }}
+        run: |
+          cd testbench
+          python generate_test_data.py
+          coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_file_read_write.py
+          coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fft.py
+          coverage run --source=bifrost.ring,bifrost,bifrost.pipeline your_first_block.py
+          python download_breakthrough_listen_data.py -y
+          coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fdmt.py ./testdata/pulsars/blc0_guppi_57407_61054_PSR_J1840%2B5640_0004.fil
+          coverage xml
       - name: "Upload Coverage"
         env:
           UNITTEST_OS: ${{ matrix.os }}
           UNITTEST_PY: ${{ matrix.python-version }}
         if: ${{ matrix.os == 'self-hosted' && matrix.python-version == '3.8' }}
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
         with:
-          directory: ./test/
+          files: ./test/coverage.xml, ./testbench/coverage.xml
           env_vars: UNITTEST_OS,UNITTEST_PY
           fail_ci_if_error: false
           verbose: true
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@ doc
 version.py
 /python/bifrost/version/__init__.py
 *_generated.py
+*_typehints.py
 .log*.txt
 test/data/
 *.bin

diff --git a/.mypy.ini b/.mypy.ini
@@ -0,0 +1,4 @@
+[mypy]
+
+[mypy-scipy.fftpack.*]
+ignore_missing_imports = True
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,14 @@
+0.11.0
+ * Dropped support for Python2
+ * Merged the ibverb-support branch that brings in a new, more flexible packet capture interface
+ * Improved documentation about packet capture
+ * Improved documentation about CUDA support
+ * Improved documentation build process
+ * Various fixes for CUDA 12.x releases
+ * Added support for gcc 13
+ * Changed the file locking so that the user is warned if a block is waiting to acquire the lock
+ * Various deprecation cleanups
+
 0.10.1
  * Cleaned up the Makefile outputs
  * Added a disk cache for bifrost.map calls
@@ -8,6 +19,7 @@
  * Added set_stream and get_stream to bifrost.device to help control which CUDA stream is used
  * Added bifrost.device.ExternalStream as a context manager to help with mixing Bifrost and cupy/pycuda
  * Fixed a problem calling bifrost.reduce on a slice of an array
+ * Added the astype() method to `bifrost.ndarray`
 
 0.10.0
  * Switched over to an autotools-based build system

diff --git a/Makefile.in b/Makefile.in
@@ -7,8 +7,12 @@ DAT_DIR = share
 SRC_DIR = src
 
 HAVE_PYTHON = @HAVE_PYTHON@
+
 HAVE_DOCKER = @HAVE_DOCKER@
 
+CAN_BUILD_CXX_DOCS = @HAVE_CXX_DOCS@
+CAN_BUILD_PYTHON_DOCS = @HAVE_PYTHON_DOCS@
+
 BIFROST_PYTHON_DIR = python
 
 all: libbifrost python
@@ -18,10 +22,23 @@ libbifrost:
 	$(MAKE) -C $(SRC_DIR) all
 .PHONY: libbifrost
 
+check:
+ifeq ($(HAVE_PYTHON),1)
+	MYPYPATH=$(BIFROST_PYTHON_DIR) mypy --follow-imports=silent \
+	    python/bifrost/blocks/detect.py \
+	    python/bifrost/blocks/quantize.py \
+	    python/bifrost/blocks/unpack.py \
+	    python/bifrost/sigproc.py \
+	    python/bifrost/sigproc2.py \
+	    test/test_sigproc.py \
+	    testbench/test_fft_detect.py
+endif
+.PHONY: check
+
 test:
 	#$(MAKE) -C $(SRC_DIR) test
 ifeq ($(HAVE_PYTHON),1)
-	cd test && ./download_test_data.sh ; python -m unittest discover
+	cd test && ./download_test_data.sh ; @PYTHON@ -m unittest discover -v
 endif
 .PHONY: test
 clean:
@@ -63,8 +80,13 @@ ifeq ($(HAVE_PYTHON),1)
 endif
 .PHONY: uninstall
 
-doc: $(INC_DIR)/bifrost/*.h Doxyfile
+doc: $(INC_DIR)/bifrost/*.h Doxyfile docs/source/*.rst docs/source/*.py
+ifeq ($(CAN_BUILD_CXX_DOCS),1)
 	@DX_DOXYGEN@ Doxyfile
+endif
+ifeq ($(CAN_BUILD_PYTHON_DOCS),1)
+	$(MAKE) -C docs singlehtml
+endif
 .PHONY: doc
 
 python: libbifrost

diff --git a/README.md b/README.md
@@ -95,10 +95,20 @@ print "All done"
 **For a quick demo which you can run in-browser without installation,
 go to the following [link](https://colab.research.google.com/github/ledatelescope/bifrost/blob/master/BifrostDemo.ipynb).**
 
+### CUDA
+
+CUDA is available at https://developer.nvidia.com/cuda-downloads. You can check the
+["Getting Started guide"](http://ledatelescope.github.io/bifrost/Getting-started-guide.html) 
+in the docs to see which versions of the CUDA toolkit have been confirmed to work with Bifrost. 
+
 ### C Dependencies
 
+If using Ubuntu or another Debian-based linux distribution:
+
     $ sudo apt-get install exuberant-ctags
 
+Otherwise check https://ctags.sourceforge.net/ for install instructions.
+
 ### Python Dependencies
 
  * numpy
@@ -171,8 +181,17 @@ your machine.
 
 ### Building the Docs from Scratch
 
-Install sphinx and breathe using pip, and also install Doxygen.
+Install breathe using pip:
+
+    $ sudo pip install breathe sphinx
+
+Also install Doxygen using your package manager. 
+In Ubuntu, for example:
+
+    $ sudo apt-get install doxygen
 
+Otherwise check https://www.doxygen.nl/ for Doxygen install instructions.
+
 Doxygen documentation can be generated by running:
 
     $ make doc