From 16be285db6a783a5aea97cc3da86b38bda4f726f Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:26:47 -0700 Subject: [PATCH 01/56] Round 1. --- .github/workflows/main.yml | 2 -- python/bifrost/__init__.py | 3 +-- python/bifrost/address.py | 23 ++++------------------ python/bifrost/affinity.py | 5 +---- python/bifrost/block.py | 8 +------- python/bifrost/block_chainer.py | 4 +--- python/bifrost/core.py | 5 +---- python/bifrost/device.py | 12 ++--------- python/bifrost/dtype.py | 7 ++----- python/bifrost/fdmt.py | 5 +---- python/bifrost/fft.py | 5 +---- python/bifrost/fir.py | 7 ++----- python/bifrost/guppi_raw.py | 13 +++--------- python/bifrost/libbifrost.py | 11 ++--------- python/bifrost/linalg.py | 3 --- python/bifrost/memory.py | 5 +---- python/bifrost/ndarray.py | 35 ++++++--------------------------- python/bifrost/pipeline.py | 8 +------- python/bifrost/portaudio.py | 4 +--- python/bifrost/proclog.py | 22 +++------------------ python/bifrost/psrdada.py | 4 +--- python/bifrost/quantize.py | 5 +---- python/bifrost/reduce.py | 5 +---- python/bifrost/ring.py | 25 ++++------------------- python/bifrost/ring2.py | 33 ++++++------------------------- python/bifrost/romein.py | 5 +---- python/bifrost/sigproc.py | 18 +++-------------- python/bifrost/sigproc2.py | 18 +++-------------- python/bifrost/transpose.py | 5 +---- python/bifrost/udp_socket.py | 5 +---- python/bifrost/udp_transmit.py | 2 +- python/bifrost/unpack.py | 5 +---- 32 files changed, 58 insertions(+), 259 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d67d35044..5b2df2bea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,8 +22,6 @@ jobs: os: [self-hosted, ubuntu-latest, macos-latest] python-version: ['3.8', '3.10'] include: - - os: ubuntu-20.04 - python-version: '2.7' - os: ubuntu-20.04 python-version: '3.6' - os: macos-latest diff --git a/python/bifrost/__init__.py b/python/bifrost/__init__.py index 609e935b3..68589ed9d 100644 --- a/python/bifrost/__init__.py +++ b/python/bifrost/__init__.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,7 +31,6 @@ """ # TODO: Decide how to organise the namespace -from __future__ import print_function, absolute_import from bifrost import core, memory, affinity, ring, block, address, udp_socket from bifrost import pipeline diff --git a/python/bifrost/address.py b/python/bifrost/address.py index 73eec72e0..5b4bff2ff 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info > (3,): - long = int - from bifrost.libbifrost import _bf, _check, _get, BifrostObject import ctypes @@ -41,12 +35,8 @@ class Address(BifrostObject): def __init__(self, address, port, family=None): - try: - address = address.encode() - except AttributeError: - # Python2 catch - pass - assert(isinstance(port, (int, long))) + address = address.encode() + assert(isinstance(port, int)) if family is None: family = AF_UNSPEC BifrostObject.__init__( @@ -66,11 +56,6 @@ def address(self): buflen = 128 buf = ctypes.create_string_buffer(buflen) _check(_bf.bfAddressGetString(self.obj, buflen, buf)) - try: - value = buf.value.decode() - except AttributeError: - # Python2 catch - value = buf.value - return value + return buf.value.decode() def __str__(self): return "%s:%i" % (self.address, self.port) diff --git a/python/bifrost/affinity.py b/python/bifrost/affinity.py index 99e67e762..854db90f3 100644 --- a/python/bifrost/affinity.py +++ b/python/bifrost/affinity.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, _get, _array from bifrost import telemetry diff --git a/python/bifrost/block.py b/python/bifrost/block.py index 9714188ce..c92282750 100644 --- a/python/bifrost/block.py +++ b/python/bifrost/block.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,12 +31,6 @@ of a simple transform which works on a span by span basis. """ -# Python2 compatibility -from __future__ import print_function, division, absolute_import -import sys -if sys.version_info < (3,): - range = xrange - import json import threading import time diff --git a/python/bifrost/block_chainer.py b/python/bifrost/block_chainer.py index b9332e729..e331851a0 100644 --- a/python/bifrost/block_chainer.py +++ b/python/bifrost/block_chainer.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import print_function - import bifrost from bifrost import telemetry diff --git a/python/bifrost/core.py b/python/bifrost/core.py index 8da672d3c..41257b232 100644 --- a/python/bifrost/core.py +++ b/python/bifrost/core.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf from bifrost import telemetry diff --git a/python/bifrost/device.py b/python/bifrost/device.py index dc3ec5771..3211b258f 100644 --- a/python/bifrost/device.py +++ b/python/bifrost/device.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from ctypes import c_ulong, pointer as c_pointer from bifrost.libbifrost import _bf, _check, _get @@ -38,12 +35,7 @@ def set_device(device): if isinstance(device, int): _check(_bf.bfDeviceSet(device)) else: - try: - device = device.encode() - except AttributeError: - # Python2 catch - pass - _check(_bf.bfDeviceSetById(device)) + _check(_bf.bfDeviceSetById(device.encode())) def get_device(): return _get(_bf.bfDeviceGet) diff --git a/python/bifrost/dtype.py b/python/bifrost/dtype.py index 788a6b160..93de54d86 100644 --- a/python/bifrost/dtype.py +++ b/python/bifrost/dtype.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,9 +41,6 @@ """ -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED import numpy as np @@ -179,4 +176,4 @@ def bifrost2string(dtype): if dtype_str is None: raise ValueError("Could not convert dtype integer to string. Value not understood.") else: - return dtype_str \ No newline at end of file + return dtype_str diff --git a/python/bifrost/fdmt.py b/python/bifrost/fdmt.py index 9a902ce09..20efb7be0 100644 --- a/python/bifrost/fdmt.py +++ b/python/bifrost/fdmt.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, _get, BifrostObject, _string2space from bifrost.ndarray import asarray diff --git a/python/bifrost/fft.py b/python/bifrost/fft.py index 67777f2fa..67f2c60dc 100644 --- a/python/bifrost/fft.py +++ b/python/bifrost/fft.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, _get, BifrostObject from bifrost.ndarray import asarray import ctypes diff --git a/python/bifrost/fir.py b/python/bifrost/fir.py index e7873c4ba..f868a1746 100644 --- a/python/bifrost/fir.py +++ b/python/bifrost/fir.py @@ -1,6 +1,6 @@ -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, BifrostObject, _string2space from bifrost.ndarray import asarray diff --git a/python/bifrost/guppi_raw.py b/python/bifrost/guppi_raw.py index 9e9341f84..bde91560f 100644 --- a/python/bifrost/guppi_raw.py +++ b/python/bifrost/guppi_raw.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,9 +54,6 @@ """ -# Python2 compatibility -from __future__ import division - from bifrost import telemetry telemetry.track_module() @@ -70,12 +67,8 @@ def read_header(f): if len(record) < RECORD_LEN: raise IOError("EOF reached in middle of header") - try: - record = record.decode() - except AttributeError: - # Python2 catch - pass - if record.startswith(b'END'): + record = record.decode() + if record.startswith('END'): break key, val = record.split('=', 1) key, val = key.strip(), val.strip() diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index 08eda69c1..ca643688c 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -34,9 +34,6 @@ # instance instead of LP_s # E.g., _bf.bfRingSequenceGetName() [should be ] -# Python2 compatibility -from __future__ import absolute_import - import ctypes import bifrost.libbifrost_generated as _bf bf = _bf # Public access to library @@ -110,11 +107,7 @@ def _array(size_or_vals, dtype=None): elif isinstance(vals[0], float): dtype = ctypes.c_double elif isinstance(vals[0], str): - try: - vals = [val.encode() for val in vals] - except AttributeError: - # Python2 catch - pass + vals = [val.encode() for val in vals] dtype = ctypes.c_char_p elif isinstance(vals[0], _bf.BFarray): dtype = ctypes.POINTER(_bf.BFarray) diff --git a/python/bifrost/linalg.py b/python/bifrost/linalg.py index e8029349f..8b0a7116d 100644 --- a/python/bifrost/linalg.py +++ b/python/bifrost/linalg.py @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray diff --git a/python/bifrost/memory.py b/python/bifrost/memory.py index dc884e00b..2747c8919 100644 --- a/python/bifrost/memory.py +++ b/python/bifrost/memory.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, _get, _string2space import ctypes diff --git a/python/bifrost/ndarray.py b/python/bifrost/ndarray.py index d2ef24abf..2b45f2352 100644 --- a/python/bifrost/ndarray.py +++ b/python/bifrost/ndarray.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -36,12 +36,6 @@ """ -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange - import ctypes import numpy as np from bifrost.memory import raw_malloc, raw_free, raw_get_space, space_accessible @@ -52,11 +46,6 @@ from bifrost.Space import Space from bifrost.libbifrost_generated import struct_BFarray_ -try: - from bifrost._pypy3_compat import PyMemoryView_FromMemory -except ImportError: - pass - from bifrost import telemetry telemetry.track_module() @@ -75,23 +64,11 @@ def _address_as_buffer(address, nbyte, readonly=False): # Note: This works as a buffer in regular python and pypy # Note: int_asbuffer is undocumented; see here: # https://mail.scipy.org/pipermail/numpy-discussion/2008-January/030938.html - try: - int_asbuffer = ctypes.pythonapi.PyMemoryView_FromMemory - int_asbuffer.restype = ctypes.py_object - int_asbuffer.argtypes = (ctypes.c_void_p, ctypes.c_ssize_t, ctypes.c_int) - return int_asbuffer(address, nbyte, 0x100 if readonly else 0x200) - except AttributeError: - try: - # Python2 catch - return np.core.multiarray.int_asbuffer(address, - nbyte, - readonly=readonly, - check=False) - except AttributeError: - # PyPy3 catch - int_asbuffer = PyMemoryView_FromMemory - return int_asbuffer(address, nbyte, 0x100 if readonly else 0x200) - + int_asbuffer = ctypes.pythonapi.PyMemoryView_FromMemory + int_asbuffer.restype = ctypes.py_object + int_asbuffer.argtypes = (ctypes.c_void_p, ctypes.c_ssize_t, ctypes.c_int) + return int_asbuffer(address, nbyte, 0x100 if readonly else 0x200) + def asarray(arr, space=None): if isinstance(arr, ndarray) and (space is None or space == arr.bf.space): return arr diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index 96bd6614a..adab8a71d 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function -import sys -if sys.version_info < (3,): - range = xrange - import threading try: import queue diff --git a/python/bifrost/portaudio.py b/python/bifrost/portaudio.py index 5053a5ea0..89f5a612e 100644 --- a/python/bifrost/portaudio.py +++ b/python/bifrost/portaudio.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,8 +31,6 @@ # Ubuntu 16.04: # sudo apt-get install portaudio19-dev -from __future__ import print_function - import ctypes import atexit from threading import Lock diff --git a/python/bifrost/proclog.py b/python/bifrost/proclog.py index 8b0800b6a..46a7834fc 100644 --- a/python/bifrost/proclog.py +++ b/python/bifrost/proclog.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function, absolute_import -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.libbifrost import _bf, _check, BifrostObject import os @@ -43,13 +37,8 @@ class ProcLog(BifrostObject): def __init__(self, name): - try: - name = name.encode('utf-8') - except AttributeError: - # Python2 catch - pass BifrostObject.__init__( - self, _bf.bfProcLogCreate, _bf.bfProcLogDestroy, name) + self, _bf.bfProcLogCreate, _bf.bfProcLogDestroy, name.encode()) def update(self, contents): """Updates (replaces) the contents of the log contents: string or dict containing data to write to the log @@ -59,12 +48,7 @@ def update(self, contents): if isinstance(contents, dict): contents = '\n'.join(['%s : %s' % item for item in contents.items()]) - try: - contents = contents.encode() - except AttributeError: - # Python2 catch - pass - _check(_bf.bfProcLogUpdate(self.obj, contents)) + _check(_bf.bfProcLogUpdate(self.obj, contents.encode())) def _multi_convert(value): """ diff --git a/python/bifrost/psrdada.py b/python/bifrost/psrdada.py index bca88e19c..7b946e495 100644 --- a/python/bifrost/psrdada.py +++ b/python/bifrost/psrdada.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -39,8 +39,6 @@ libtest_la_LDFLAGS = -version-info 0:0:0 """ -from __future__ import absolute_import, print_function - import bifrost.libpsrdada_generated as _dada import numpy as np from bifrost.ndarray import _address_as_buffer diff --git a/python/bifrost/quantize.py b/python/bifrost/quantize.py index 533b686d4..456c513ba 100644 --- a/python/bifrost/quantize.py +++ b/python/bifrost/quantize.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray diff --git a/python/bifrost/reduce.py b/python/bifrost/reduce.py index 05b1cf8ac..601487943 100644 --- a/python/bifrost/reduce.py +++ b/python/bifrost/reduce.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray diff --git a/python/bifrost/ring.py b/python/bifrost/ring.py index e4335e1f6..6f72c763f 100644 --- a/python/bifrost/ring.py +++ b/python/bifrost/ring.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function, absolute_import - from bifrost.libbifrost import _bf, _check, _get, BifrostObject, _string2space, _space2string, EndOfDataStop #from GPUArray import GPUArray from bifrost.DataType import DataType @@ -52,16 +49,11 @@ def __init__(self, space='system', name=None, core=None): if name is None: name = str(uuid4()) name = _slugify(name) - try: - name = name.encode() - except AttributeError: - # Python2 catch - pass space = _string2space(space) #self.obj = None #self.obj = _get(_bf.bfRingCreate(name=name, space=space), retarg=0) BifrostObject.__init__( - self, _bf.bfRingCreate, _bf.bfRingDestroy, name, space) + self, _bf.bfRingCreate, _bf.bfRingDestroy, name.encode(), space) if core is not None: try: _check( _bf.bfRingSetAffinity(self.obj, @@ -208,24 +200,15 @@ def __init__(self, ring, name="", time_tag=-1, header="", nringlet=1): if isinstance(header, np.ndarray): header = header.ctypes.data elif isinstance(header, str): - try: - header = header.encode() - except AttributeError: - # Python2 catch - pass + header = header.encode() #print("hdr:", header_size, type(header)) name = str(name) offset_from_head = 0 self.obj = _bf.BFwsequence() - try: - name = name.encode() - except AttributeError: - # Python2 catch - pass _check(_bf.bfRingSequenceBegin( self.obj, ring.obj, - name, + name.encode(), time_tag, header_size, header, diff --git a/python/bifrost/ring2.py b/python/bifrost/ring2.py index 1ff94b2d9..06ef49ac3 100644 --- a/python/bifrost/ring2.py +++ b/python/bifrost/ring2.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,8 +28,6 @@ # TODO: Some of this code has gotten a bit hacky # Also consider merging some of the logic into the backend -from __future__ import print_function, absolute_import - from bifrost.libbifrost import _bf, _check, _get, BifrostObject, _string2space, EndOfDataStop from bifrost.DataType import DataType from bifrost.ndarray import ndarray, _address_as_buffer @@ -91,13 +89,8 @@ def __init__(self, space='system', name=None, owner=None, core=None): name = 'ring_%i' % Ring.instance_count Ring.instance_count += 1 name = _slugify(name) - try: - name = name.encode() - except AttributeError: - # Python2 catch - pass BifrostObject.__init__(self, _bf.bfRingCreate, _bf.bfRingDestroy, - name, _string2space(self.space)) + name.encode(), _string2space(self.space)) if core is not None: try: _check( _bf.bfRingSetAffinity(self.obj, @@ -122,12 +115,7 @@ def resize(self, contiguous_bytes, total_bytes=None, nringlet=1): @property def name(self): n = _get(_bf.bfRingGetName, self.obj) - try: - n = n.decode() - except AttributeError: - # Python2 catch - pass - return n + return n.decode() @property def core(self): return _get(_bf.bfRingGetAffinity, self.obj) @@ -212,11 +200,7 @@ def tensor(self): # TODO: This shouldn't be public nringlet = reduce(lambda x, y: x * y, ringlet_shape, 1) frame_nelement = reduce(lambda x, y: x * y, frame_shape, 1) dtype = header['_tensor']['dtype'] - try: - dtype = dtype.decode() - except AttributeError: - # Python2 catch - pass + dtype = dtype.decode() nbit = DataType(dtype).itemsize_bits assert(nbit % 8 == 0) frame_nbyte = frame_nelement * nbit // 8 @@ -260,13 +244,8 @@ def __init__(self, ring, header, gulp_nframe, buf_nframe): offset_from_head = 0 # TODO: How to allow time_tag to be optional? Probably need to plumb support through to backend. self.obj = _bf.BFwsequence() - try: - hname = header['name'].encode() - hstr = header_str.encode() - except AttributeError: - # Python2 catch - hname = header['name'] - hstr = header_str + hname = header['name'].encode() + hstr = header_str.encode() _check(_bf.bfRingSequenceBegin( self.obj, ring.obj, diff --git a/python/bifrost/romein.py b/python/bifrost/romein.py index 757dd2a0d..a89c008c6 100644 --- a/python/bifrost/romein.py +++ b/python/bifrost/romein.py @@ -1,5 +1,5 @@ -# Copyright (c) 2018-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2018-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray diff --git a/python/bifrost/sigproc.py b/python/bifrost/sigproc.py index cfbd2f21f..8f4a6f145 100644 --- a/python/bifrost/sigproc.py +++ b/python/bifrost/sigproc.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -52,8 +52,6 @@ data: [time][pol][nbit] (General case: [time][if/pol][chan][nbit]) """ -from __future__ import print_function - import struct import warnings import numpy as np @@ -134,12 +132,7 @@ def _header_write_string(file_object, key): """Writes a single key name to the header, which will be followed by the value""" file_object.write(struct.pack('=i', len(key))) - try: - key = key.encode() - except AttributeError: - # Catch for Python2 - pass - file_object.write(key) + file_object.write(key.encode()) def _header_write_value(file_object, key, value): """Writes a single parameter value to the header""" @@ -160,12 +153,7 @@ def _header_read_one_parameter(file_object): if length <= 0 or length >= 80: return None s = file_object.read(length) - try: - s = s.decode() - except AttributeError: - # Python2 catch - pass - return s + return s.decode() def _write_header(hdr, file_object): """write the entire header to the current position of a file""" diff --git a/python/bifrost/sigproc2.py b/python/bifrost/sigproc2.py index a85956091..4f1e6d4a3 100644 --- a/python/bifrost/sigproc2.py +++ b/python/bifrost/sigproc2.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,8 +53,6 @@ # See here for details of the different data formats: # https://github.com/SixByNine/sigproc -from __future__ import print_function, division - import struct import warnings import numpy as np @@ -147,12 +145,7 @@ def machine2id(name): def _header_write_string(f, key): f.write(struct.pack('=i', len(key))) - try: - key = key.encode('ascii') - except AttributeError: - # Catch for Python2 - pass - f.write(key) + f.write(key.encode()) def _header_write(f, key, value, fmt=None): if fmt is not None: pass @@ -172,12 +165,7 @@ def _header_read(f): if length < 0 or length >= 80: return None s = f.read(length) - try: - s = s.decode() - except AttributeError: - # Python2 catch - pass - return s + return s.decode() def write_header(hdr, f): _header_write_string(f, "HEADER_START") diff --git a/python/bifrost/transpose.py b/python/bifrost/transpose.py index 6ff907963..e84245dbb 100644 --- a/python/bifrost/transpose.py +++ b/python/bifrost/transpose.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray diff --git a/python/bifrost/udp_socket.py b/python/bifrost/udp_socket.py index 2f04ef784..161e02d13 100644 --- a/python/bifrost/udp_socket.py +++ b/python/bifrost/udp_socket.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # **TODO: Write tests for this class -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check, _get, BifrostObject from bifrost import telemetry diff --git a/python/bifrost/udp_transmit.py b/python/bifrost/udp_transmit.py index cb4791281..5bb4c0060 100644 --- a/python/bifrost/udp_transmit.py +++ b/python/bifrost/udp_transmit.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-202, The Bifrost Authors. All rights reserved. # Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/python/bifrost/unpack.py b/python/bifrost/unpack.py index c41f19d9a..f25dd7777 100644 --- a/python/bifrost/unpack.py +++ b/python/bifrost/unpack.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import - from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray From b82abaadda858e05e926cc185785a7725653dc54 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:31:47 -0700 Subject: [PATCH 02/56] Round 2. --- python/bifrost/DataType.py | 12 ++---------- python/bifrost/telemetry/__init__.py | 7 ++----- python/bifrost/telemetry/__main__.py | 7 ++----- python/bifrost/version/__main__.py | 2 -- python/bifrost/views/basic_views.py | 4 +--- 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/python/bifrost/DataType.py b/python/bifrost/DataType.py index a500e22d8..f4a69b477 100644 --- a/python/bifrost/DataType.py +++ b/python/bifrost/DataType.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -38,14 +38,6 @@ cf32: 32+32-bit complex floating point """ -# Python2 compatibility -from __future__ import division, absolute_import -import sys -string_types = (str,) -if sys.version_info < (3,): - range = xrange - string_types = (basestring,) - from bifrost.libbifrost import _bf from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED import numpy as np @@ -126,7 +118,7 @@ def is_vector_structure(dtype): class DataType(object): # Note: Default of None results in default Numpy type (np.float) def __init__(self, t=None): - if isinstance(t, string_types): + if isinstance(t, str): for i, char in enumerate(t): if char.isdigit(): break diff --git a/python/bifrost/telemetry/__init__.py b/python/bifrost/telemetry/__init__.py index 3a77c282b..0b22e6ef3 100644 --- a/python/bifrost/telemetry/__init__.py +++ b/python/bifrost/telemetry/__init__.py @@ -1,6 +1,6 @@ -# Copyright (c) 2021-2022, The Bifrost Authors. All rights reserved. -# Copyright (c) 2021-2022, The University of New Mexico. All rights reserved. +# Copyright (c) 2021-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2021-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function, division, absolute_import - import os import time import uuid diff --git a/python/bifrost/telemetry/__main__.py b/python/bifrost/telemetry/__main__.py index 7cf635575..a693ecd86 100644 --- a/python/bifrost/telemetry/__main__.py +++ b/python/bifrost/telemetry/__main__.py @@ -1,6 +1,6 @@ -# Copyright (c) 2021-2022, The Bifrost Authors. All rights reserved. -# Copyright (c) 2021-2022, The University of New Mexico. All rights reserved. +# Copyright (c) 2021-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2021-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,9 +25,6 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Python2 compatibility -from __future__ import print_function, division, absolute_import import argparse diff --git a/python/bifrost/version/__main__.py b/python/bifrost/version/__main__.py index d352987d6..d6b37cac9 100644 --- a/python/bifrost/version/__main__.py +++ b/python/bifrost/version/__main__.py @@ -26,8 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import print_function - import argparse from bifrost import __version__, __copyright__, __license__ diff --git a/python/bifrost/views/basic_views.py b/python/bifrost/views/basic_views.py index a7ba1e07a..419bdc7cf 100644 --- a/python/bifrost/views/basic_views.py +++ b/python/bifrost/views/basic_views.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, division - from bifrost.pipeline import block_view from bifrost.DataType import DataType from bifrost.units import convert_units From c65f6589bb000a675191cfc958b4edd41225de2f Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:37:21 -0700 Subject: [PATCH 03/56] Round 3. --- python/bifrost/blocks/__init__.py | 4 +--- python/bifrost/blocks/accumulate.py | 4 +--- python/bifrost/blocks/audio.py | 4 +--- python/bifrost/blocks/convert_visibilities.py | 4 +--- python/bifrost/blocks/copy.py | 4 +--- python/bifrost/blocks/correlate.py | 8 +------ python/bifrost/blocks/detect.py | 8 +------ python/bifrost/blocks/fdmt.py | 4 +--- python/bifrost/blocks/fft.py | 4 +--- python/bifrost/blocks/fftshift.py | 8 +------ python/bifrost/blocks/guppi_raw.py | 4 +--- python/bifrost/blocks/print_header.py | 3 +-- python/bifrost/blocks/psrdada.py | 4 +--- python/bifrost/blocks/quantize.py | 4 +--- python/bifrost/blocks/reduce.py | 4 +--- python/bifrost/blocks/reverse.py | 8 +------ python/bifrost/blocks/scrunch.py | 4 +--- python/bifrost/blocks/serialize.py | 8 +------ python/bifrost/blocks/sigproc.py | 8 +------ python/bifrost/blocks/transpose.py | 3 +-- python/bifrost/blocks/unpack.py | 4 +--- python/bifrost/blocks/wav.py | 23 +++---------------- 22 files changed, 24 insertions(+), 105 deletions(-) diff --git a/python/bifrost/blocks/__init__.py b/python/bifrost/blocks/__init__.py index b878d6a43..d786e2f5c 100644 --- a/python/bifrost/blocks/__init__.py +++ b/python/bifrost/blocks/__init__.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.blocks.copy import copy, CopyBlock from bifrost.blocks.transpose import transpose, TransposeBlock from bifrost.blocks.reverse import reverse, ReverseBlock diff --git a/python/bifrost/blocks/accumulate.py b/python/bifrost/blocks/accumulate.py index c405f5c64..381f9c841 100644 --- a/python/bifrost/blocks/accumulate.py +++ b/python/bifrost/blocks/accumulate.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,8 +30,6 @@ nframe times before outputting the accumulated result. """ -from __future__ import absolute_import - from bifrost.map import map as bf_map from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/audio.py b/python/bifrost/blocks/audio.py index 1db0ffbf0..f083e9418 100644 --- a/python/bifrost/blocks/audio.py +++ b/python/bifrost/blocks/audio.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import SourceBlock import bifrost.portaudio as audio diff --git a/python/bifrost/blocks/convert_visibilities.py b/python/bifrost/blocks/convert_visibilities.py index 079d351f8..2404ba20a 100644 --- a/python/bifrost/blocks/convert_visibilities.py +++ b/python/bifrost/blocks/convert_visibilities.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.map import map as bf_map from bifrost.pipeline import TransformBlock from bifrost.DataType import DataType diff --git a/python/bifrost/blocks/copy.py b/python/bifrost/blocks/copy.py index 0a6a6cf6a..4e15030c2 100644 --- a/python/bifrost/blocks/copy.py +++ b/python/bifrost/blocks/copy.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import TransformBlock from bifrost.ndarray import copy_array diff --git a/python/bifrost/blocks/correlate.py b/python/bifrost/blocks/correlate.py index fb29d5a66..246487729 100644 --- a/python/bifrost/blocks/correlate.py +++ b/python/bifrost/blocks/correlate.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.pipeline import TransformBlock from bifrost.linalg import LinAlg diff --git a/python/bifrost/blocks/detect.py b/python/bifrost/blocks/detect.py index 4ac1a81ab..30547de7e 100644 --- a/python/bifrost/blocks/detect.py +++ b/python/bifrost/blocks/detect.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,12 +24,6 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange from bifrost.map import map as bf_map from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/fdmt.py b/python/bifrost/blocks/fdmt.py index a86e99102..65b0e1e0c 100644 --- a/python/bifrost/blocks/fdmt.py +++ b/python/bifrost/blocks/fdmt.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import TransformBlock from bifrost.fdmt import Fdmt from bifrost.units import convert_units diff --git a/python/bifrost/blocks/fft.py b/python/bifrost/blocks/fft.py index 2e2e7a12d..42fa4c7d2 100644 --- a/python/bifrost/blocks/fft.py +++ b/python/bifrost/blocks/fft.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import TransformBlock from bifrost.fft import Fft from bifrost.units import transform_units diff --git a/python/bifrost/blocks/fftshift.py b/python/bifrost/blocks/fftshift.py index 3a0a19b3a..0cf8fc400 100644 --- a/python/bifrost/blocks/fftshift.py +++ b/python/bifrost/blocks/fftshift.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.map import map as bf_map from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/guppi_raw.py b/python/bifrost/blocks/guppi_raw.py index 870a2c7cb..9085cb7dd 100644 --- a/python/bifrost/blocks/guppi_raw.py +++ b/python/bifrost/blocks/guppi_raw.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import SourceBlock import bifrost.guppi_raw as guppi_raw diff --git a/python/bifrost/blocks/print_header.py b/python/bifrost/blocks/print_header.py index 919a696fe..ae46495bb 100644 --- a/python/bifrost/blocks/print_header.py +++ b/python/bifrost/blocks/print_header.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,7 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, print_function import pprint from bifrost.pipeline import SinkBlock diff --git a/python/bifrost/blocks/psrdada.py b/python/bifrost/blocks/psrdada.py index ca159d29e..6965e7f3f 100644 --- a/python/bifrost/blocks/psrdada.py +++ b/python/bifrost/blocks/psrdada.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.pipeline import SourceBlock from bifrost.Space import Space from bifrost.psrdada import Hdu diff --git a/python/bifrost/blocks/quantize.py b/python/bifrost/blocks/quantize.py index 309c3950d..847885c57 100644 --- a/python/bifrost/blocks/quantize.py +++ b/python/bifrost/blocks/quantize.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.quantize import quantize as bf_quantize from bifrost.pipeline import TransformBlock from bifrost.DataType import DataType diff --git a/python/bifrost/blocks/reduce.py b/python/bifrost/blocks/reduce.py index 51ff2b001..93a884d00 100644 --- a/python/bifrost/blocks/reduce.py +++ b/python/bifrost/blocks/reduce.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,8 +28,6 @@ # TODO: Consider merging with detect_block # Seems easy, but may end up somewhat complicated -from __future__ import absolute_import - from bifrost.reduce import reduce as bf_reduce from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/reverse.py b/python/bifrost/blocks/reverse.py index cb088dd6f..4641c7e0b 100644 --- a/python/bifrost/blocks/reverse.py +++ b/python/bifrost/blocks/reverse.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.map import map as bf_map from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/scrunch.py b/python/bifrost/blocks/scrunch.py index 9956f2bb0..0068b6cbd 100644 --- a/python/bifrost/blocks/scrunch.py +++ b/python/bifrost/blocks/scrunch.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,8 +28,6 @@ # TODO: This is a bit hacky and inflexible, and has no CUDA backend yet # **DEPRECATE it in favour of ReduceBlock -from __future__ import absolute_import - from bifrost.pipeline import TransformBlock from copy import deepcopy diff --git a/python/bifrost/blocks/serialize.py b/python/bifrost/blocks/serialize.py index 4bab53fac..04cbc03f3 100644 --- a/python/bifrost/blocks/serialize.py +++ b/python/bifrost/blocks/serialize.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import, print_function -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.pipeline import SinkBlock, SourceBlock import os import warnings diff --git a/python/bifrost/blocks/sigproc.py b/python/bifrost/blocks/sigproc.py index 397f1c2fb..ad9675f74 100644 --- a/python/bifrost/blocks/sigproc.py +++ b/python/bifrost/blocks/sigproc.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import, print_function -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.pipeline import SourceBlock, SinkBlock import bifrost.sigproc2 as sigproc from bifrost.DataType import DataType diff --git a/python/bifrost/blocks/transpose.py b/python/bifrost/blocks/transpose.py index f1ffc8909..a9df042d4 100644 --- a/python/bifrost/blocks/transpose.py +++ b/python/bifrost/blocks/transpose.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import from bifrost.transpose import transpose as bf_transpose from bifrost.memory import space_accessible from bifrost.pipeline import TransformBlock diff --git a/python/bifrost/blocks/unpack.py b/python/bifrost/blocks/unpack.py index ac9645cf8..4119736ad 100644 --- a/python/bifrost/blocks/unpack.py +++ b/python/bifrost/blocks/unpack.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import - from bifrost.unpack import unpack as bf_unpack from bifrost.pipeline import TransformBlock from bifrost.DataType import DataType diff --git a/python/bifrost/blocks/wav.py b/python/bifrost/blocks/wav.py index 7f6966fec..2c045681c 100644 --- a/python/bifrost/blocks/wav.py +++ b/python/bifrost/blocks/wav.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,12 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info < (3,): - range = xrange - from bifrost.pipeline import SourceBlock, SinkBlock from bifrost.DataType import DataType from bifrost.units import convert_units @@ -43,21 +37,10 @@ def wav_read_chunk_desc(f): id_, size, fmt = struct.unpack('<4sI4s', f.read(12)) - try: - id_ = id_.decode() - fmt = fmt.decode() - except AttributeError: - # Catch for Python2 - pass - return id_, size, fmt + return id_.decode(), size, fmt.decode() def wav_read_subchunk_desc(f): id_, size = struct.unpack('<4sI', f.read(8)) - try: - id_ = id_.decode() - except AttributeError: - # Catch for Python2 - pass - return id_, size + return id_.decode(), size def wav_read_subchunk_fmt(f, size): assert(size >= 16) packed = f.read(16) From 6e4bfeb46ca94083af4189db61d1f9adb49915fb Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:41:02 -0700 Subject: [PATCH 04/56] Round 4. --- python/bifrost/addon/leda/bandfiles.py | 10 ++--- python/bifrost/addon/leda/blocks.py | 10 +---- python/bifrost/addon/leda/make_header.py | 53 +++++++++++------------- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/python/bifrost/addon/leda/bandfiles.py b/python/bifrost/addon/leda/bandfiles.py index e83dee2ca..47268f0b0 100644 --- a/python/bifrost/addon/leda/bandfiles.py +++ b/python/bifrost/addon/leda/bandfiles.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,8 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import print_function - import os, sys sys.path.append('..') @@ -48,9 +46,9 @@ def extract_obs_offset_from_name(fname): return int(os.path.basename(fname)[20:36]) def extract_obs_offset_in_file(fname): - f = open(fname, 'rb') - headerstr = f.read(DADA_HEADER_SIZE) - f.close() + with open(fname, 'rb') as f: + headerstr = f.read(DADA_HEADER_SIZE) + headerstr = headerstr.decode() if len(headerstr) < DADA_HEADER_SIZE: return "UNKNOWN" for line in headerstr.split('\n'): key, value = line.split() diff --git a/python/bifrost/addon/leda/blocks.py b/python/bifrost/addon/leda/blocks.py index 9980bc88d..6c72c6b31 100644 --- a/python/bifrost/addon/leda/blocks.py +++ b/python/bifrost/addon/leda/blocks.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,12 +29,6 @@ This file contains blocks specific to LEDA-OVRO. """ -# Python2 compatibility -from __future__ import print_function -import sys -if sys.version_info < (3,): - range = xrange - import os import bandfiles import bifrost @@ -120,7 +114,7 @@ def main(self, input_rings, output_rings): print("Opening", f.name) - with open(f.name,'rb') as ifile: + with open(f.name, 'rb') as ifile: ifile.read(self.HEADER_SIZE) ohdr["cfreq"] = f.freq diff --git a/python/bifrost/addon/leda/make_header.py b/python/bifrost/addon/leda/make_header.py index 26f009440..afd279385 100644 --- a/python/bifrost/addon/leda/make_header.py +++ b/python/bifrost/addon/leda/make_header.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,8 +34,6 @@ Makes header.txt files that is used by corr2uvfit and DuCT. """ -from __future__ import print_function, division - import numpy as np import os, sys, ephem, datetime from dateutil import tz @@ -72,9 +70,9 @@ def generate_info(self): based on what's in the header. For the rest, call them UNKNOWN. """ - f = open(self.filename, 'rb') - headerstr = f.read(self.DEFAULT_HEADER_SIZE) - f.close() + with open(self.filename, 'rb') as f: + headerstr = f.read(self.DEFAULT_HEADER_SIZE) + headerstr = headerstr.decode() header = {} for line in headerstr.split('\n'): @@ -270,28 +268,27 @@ def make_header(filename, write=True, warn=True, size=None): else: n_scans = str(int(header_params['N_SCANS'])) if write: # This format is used by corr2uvfits and DuCT for transforming a DADA file. - output = open("header.txt","w") - output.write("# Generated by make_header.py\n\n") - output.write("FIELDNAME Zenith\n") - output.write("N_SCANS "+n_scans+"\n") - output.write("N_INPUTS 512\n") - output.write("N_CHANS "+str(header_params['N_CHANS'])+" # number of channels in spectrum\n") - output.write("CORRTYPE B # correlation type to use. 'C'(cross), 'B'(both), or 'A'(auto)\n") - output.write("INT_TIME "+str(header_params['INT_TIME'])+" # integration time of scan in seconds\n") - output.write("FREQCENT "+str(header_params['FREQCENT'])+" # observing center freq in MHz\n") - output.write("BANDWIDTH "+str(header_params['BANDWIDTH'])+" # total bandwidth in MHz\n") - output.write("# To phase to the zenith, these must be the HA, RA and Dec of the zenith.\n") - output.write("HA_HRS 0.000000 # the RA of the desired phase centre (hours)\n") - output.write("RA_HRS "+header_params['RA_HRS']+" # the RA of the desired phase centre (hours)\n") - output.write("DEC_DEGS "+str(header_params['DEC_DEGS'])+" # the DEC of the desired phase centre (degs)\n") - output.write("DATE "+header_params['DATE']+" # YYYYMMDD\n") - output.write("TIME "+header_params['TIME']+" # HHMMSS\n") - output.write("LOCALTIME "+str(dada_times.localtime_str)+"\n") - output.write("LST "+str(dada_times.lst)+"\n") - output.write("INVERT_FREQ 0 # 1 if the freq decreases with channel number\n") - output.write("CONJUGATE 1 # conjugate the raw data to fix sign convention problem if necessary\n") - output.write("GEOM_CORRECT 0\n") - output.close() + with open('header.txt', 'w') as output: + output.write("# Generated by make_header.py\n\n") + output.write("FIELDNAME Zenith\n") + output.write("N_SCANS "+n_scans+"\n") + output.write("N_INPUTS 512\n") + output.write("N_CHANS "+str(header_params['N_CHANS'])+" # number of channels in spectrum\n") + output.write("CORRTYPE B # correlation type to use. 'C'(cross), 'B'(both), or 'A'(auto)\n") + output.write("INT_TIME "+str(header_params['INT_TIME'])+" # integration time of scan in seconds\n") + output.write("FREQCENT "+str(header_params['FREQCENT'])+" # observing center freq in MHz\n") + output.write("BANDWIDTH "+str(header_params['BANDWIDTH'])+" # total bandwidth in MHz\n") + output.write("# To phase to the zenith, these must be the HA, RA and Dec of the zenith.\n") + output.write("HA_HRS 0.000000 # the RA of the desired phase centre (hours)\n") + output.write("RA_HRS "+header_params['RA_HRS']+" # the RA of the desired phase centre (hours)\n") + output.write("DEC_DEGS "+str(header_params['DEC_DEGS'])+" # the DEC of the desired phase centre (degs)\n") + output.write("DATE "+header_params['DATE']+" # YYYYMMDD\n") + output.write("TIME "+header_params['TIME']+" # HHMMSS\n") + output.write("LOCALTIME "+str(dada_times.localtime_str)+"\n") + output.write("LST "+str(dada_times.lst)+"\n") + output.write("INVERT_FREQ 0 # 1 if the freq decreases with channel number\n") + output.write("CONJUGATE 1 # conjugate the raw data to fix sign convention problem if necessary\n") + output.write("GEOM_CORRECT 0\n") return header_params # If this function is called from other scripts (e.g. plot scripts) it can supply useful information From 811441aeb19cc2ee8bc7277e60383e21763ae321 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:46:34 -0700 Subject: [PATCH 05/56] Round 5. --- test/test_fir.py | 6 ++---- test/test_linalg.py | 5 +---- test/test_map.py | 7 ++----- test/test_pipeline_cpu.py | 7 ++----- test/test_print_header.py | 7 ++----- test/test_reduce.py | 4 +--- test/test_romein.py | 4 +--- test/test_scripts.py | 5 ++--- test/test_tutorial.py | 33 +++++++++++++++++++++++++++------ 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/test/test_fir.py b/test/test_fir.py index cca287c98..596158a5a 100644 --- a/test/test_fir.py +++ b/test/test_fir.py @@ -1,6 +1,6 @@ -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2020, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,8 +28,6 @@ """This set of unit tests check the functionality on the bifrost FIR filter.""" -# Python2 compatibility -from __future__ import division import ctypes import unittest diff --git a/test/test_linalg.py b/test/test_linalg.py index ae920f2ba..63c19fb80 100644 --- a/test/test_linalg.py +++ b/test/test_linalg.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # **TODO: Add tests with beta != 0 -# Python2 compatibility -from __future__ import print_function - import ctypes import unittest import numpy as np diff --git a/test/test_map.py b/test/test_map.py index 749edcb36..fb447a78f 100644 --- a/test/test_map.py +++ b/test/test_map.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,10 +30,7 @@ import unittest import numpy as np import bifrost as bf -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO from bifrost.libbifrost_generated import BF_CUDA_ENABLED diff --git a/test/test_pipeline_cpu.py b/test/test_pipeline_cpu.py index 6e1886014..0c6b6da92 100644 --- a/test/test_pipeline_cpu.py +++ b/test/test_pipeline_cpu.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,10 +31,7 @@ from bifrost.blocks import * -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO class CallbackBlock(CopyBlock): """Testing-only block which calls user-defined diff --git a/test/test_print_header.py b/test/test_print_header.py index 60d48fe31..1c0c5d944 100644 --- a/test/test_print_header.py +++ b/test/test_print_header.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,10 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO import bifrost.pipeline as bfp import bifrost.blocks as blocks diff --git a/test/test_reduce.py b/test/test_reduce.py index f6f0797c6..d1866e283 100644 --- a/test/test_reduce.py +++ b/test/test_reduce.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import division - import unittest import numpy as np import bifrost as bf diff --git a/test/test_romein.py b/test/test_romein.py index bdac0ac48..414f8fd26 100644 --- a/test/test_romein.py +++ b/test/test_romein.py @@ -1,5 +1,5 @@ -# Copyright (c) 2018-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2018-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,8 +25,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from __future__ import print_function - import unittest import numpy import bifrost diff --git a/test/test_scripts.py b/test/test_scripts.py index 9a75b08fa..9a509d972 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python -# Copyright (c) 2019, The Bifrost Authors. All rights reserved. -# Copyright (c) 2019, The University of New Mexico. All rights reserved. +# Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2019-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/test/test_tutorial.py b/test/test_tutorial.py index 346cb1908..767b2b7d2 100644 --- a/test/test_tutorial.py +++ b/test/test_tutorial.py @@ -1,13 +1,34 @@ + +# Copyright (c) 2021-2023, The Bifrost Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + """ Unit tests for the various Bifrost tutorial notebooks. """ -# Python2 compatibility -from __future__ import print_function, division, absolute_import -import sys -if sys.version_info < (3,): - range = xrange - import unittest import glob import sys From 05512e0dc10dc9c270f34acc95fe66862aa95855 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:50:11 -0700 Subject: [PATCH 06/56] Round 6. --- testbench/download_breakthrough_listen_data.py | 10 ++-------- testbench/generate_test_data.py | 7 ++----- testbench/gpuspec_simple.py | 7 ++----- testbench/test_fdmt.py | 7 ++----- testbench/test_fft.py | 7 ++----- testbench/test_fft_detect.py | 7 ++----- testbench/test_file_read_write.py | 7 ++----- testbench/test_guppi.py | 7 ++----- testbench/test_guppi_reader.py | 7 ++----- testbench/your_first_block.py | 7 ++----- 10 files changed, 20 insertions(+), 53 deletions(-) diff --git a/testbench/download_breakthrough_listen_data.py b/testbench/download_breakthrough_listen_data.py index d8127a43d..453b3861f 100755 --- a/testbench/download_breakthrough_listen_data.py +++ b/testbench/download_breakthrough_listen_data.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,12 +32,6 @@ Generate test data that can be used with a testbench """ -# Python2 compatibility -from __future__ import print_function -import sys -if sys.version_info < (3,): - input = raw_input - import os import sys import numpy as np diff --git a/testbench/generate_test_data.py b/testbench/generate_test_data.py index 14d69820f..68afca384 100755 --- a/testbench/generate_test_data.py +++ b/testbench/generate_test_data.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,9 +32,6 @@ Generate test data that can be used with a testbench """ -# Python2 compatibility -from __future__ import print_function - import os import numpy as np diff --git a/testbench/gpuspec_simple.py b/testbench/gpuspec_simple.py index c0e3733a1..836efaf36 100755 --- a/testbench/gpuspec_simple.py +++ b/testbench/gpuspec_simple.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import bifrost as bf from argparse import ArgumentParser diff --git a/testbench/test_fdmt.py b/testbench/test_fdmt.py index c5d88b976..3050a042b 100755 --- a/testbench/test_fdmt.py +++ b/testbench/test_fdmt.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,6 @@ Measure Transform (FDMT), writing the output to a PGM file. """ -# Python2 compatibility -from __future__ import print_function - import bifrost.pipeline as bfp from bifrost.blocks import read_sigproc, copy, transpose, fdmt, scrunch from bifrost import blocks diff --git a/testbench/test_fft.py b/testbench/test_fft.py index 596dbee25..d49fb2348 100755 --- a/testbench/test_fft.py +++ b/testbench/test_fft.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,6 @@ takes the FFT of the data (on the GPU no less), and then writes it to a new file. """ -# Python2 compatibility -from __future__ import print_function - import os import glob import numpy as np diff --git a/testbench/test_fft_detect.py b/testbench/test_fft_detect.py index 3ca79852c..889d82b4c 100755 --- a/testbench/test_fft_detect.py +++ b/testbench/test_fft_detect.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,6 @@ takes the FFT of the data (on the GPU no less), and then writes it to a new file. """ -# Python2 compatibility -from __future__ import print_function - import os import glob import numpy as np diff --git a/testbench/test_file_read_write.py b/testbench/test_file_read_write.py index 0161ab5bb..a60e302f3 100755 --- a/testbench/test_file_read_write.py +++ b/testbench/test_file_read_write.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,6 @@ and then writes the data to an output file. """ -# Python2 compatibility -from __future__ import print_function - import os import numpy as np import bifrost.pipeline as bfp diff --git a/testbench/test_guppi.py b/testbench/test_guppi.py index b467d7894..9be6d9f3f 100755 --- a/testbench/test_guppi.py +++ b/testbench/test_guppi.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,9 +32,6 @@ This testbench tests a guppi gpuspec reader """ -# Python2 compatibility -from __future__ import print_function - import os import glob import numpy as np diff --git a/testbench/test_guppi_reader.py b/testbench/test_guppi_reader.py index 4ad90a48d..11712f522 100755 --- a/testbench/test_guppi_reader.py +++ b/testbench/test_guppi_reader.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,9 +32,6 @@ This testbench tests a guppi gpuspec reader """ -# Python2 compatibility -from __future__ import print_function - import os import glob import numpy as np diff --git a/testbench/your_first_block.py b/testbench/your_first_block.py index ce359d958..d722dfd64 100755 --- a/testbench/your_first_block.py +++ b/testbench/your_first_block.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,6 @@ and then writes the data to an output file. """ -# Python2 compatibility -from __future__ import print_function - import os import numpy as np import bifrost.pipeline as bfp From 5fde61f9620603c9af4b21a35d560fba362e7148 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 18:55:43 -0700 Subject: [PATCH 07/56] Round 7. --- tools/getirq.py | 12 ++++-------- tools/getsiblings.py | 9 +++------ tools/like_bmon.py | 15 ++++----------- tools/like_pmap.py | 10 +++------- tools/like_ps.py | 16 ++++------------ tools/like_top.py | 24 ++++++------------------ tools/pipeline2dot.py | 10 +++------- tools/setirq.py | 12 ++++-------- 8 files changed, 31 insertions(+), 77 deletions(-) diff --git a/tools/getirq.py b/tools/getirq.py index 35d8178b4..cd426e153 100755 --- a/tools/getirq.py +++ b/tools/getirq.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import argparse from bifrost import telemetry @@ -36,9 +33,8 @@ def main(args): - fh = open('/proc/interrupts', 'r') - lines = fh.read() - fh.close() + with open('/proc/interrupts', 'r') as fh: + lines = fh.read() irqs = {} for line in lines.split('\n'): diff --git a/tools/getsiblings.py b/tools/getsiblings.py index f2e559efc..2994eb81f 100755 --- a/tools/getsiblings.py +++ b/tools/getsiblings.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import glob import argparse diff --git a/tools/like_bmon.py b/tools/like_bmon.py index e4c99dea4..e9630278f 100755 --- a/tools/like_bmon.py +++ b/tools/like_bmon.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2022, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2022, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import os import sys import glob @@ -39,10 +36,7 @@ import argparse import traceback from datetime import datetime -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO os.environ['VMA_TRACELEVEL'] = '0' from bifrost.proclog import PROCLOG_DIR, load_by_pid @@ -106,7 +100,6 @@ def get_command_line(pid): with open('/proc/%i/cmdline' % pid, 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') - fh.close() except IOError: pass return cmd diff --git a/tools/like_pmap.py b/tools/like_pmap.py index 4dd825eb6..74d0203fe 100755 --- a/tools/like_pmap.py +++ b/tools/like_pmap.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import os import re import argparse @@ -91,7 +88,6 @@ def main(args): try: fh = open('/proc/%i/numa_maps' % args.pid, 'r') numaInfo = fh.read() - fh.close() except IOError: raise RuntimeError("Cannot find NUMA memory info for PID: %i" % args.pid) diff --git a/tools/like_ps.py b/tools/like_ps.py index b708392c3..2d729db3a 100755 --- a/tools/like_ps.py +++ b/tools/like_ps.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import os import glob import argparse @@ -66,11 +63,7 @@ def get_process_details(pid): data = {'user':'', 'cpu':0.0, 'mem':0.0, 'etime':'00:00', 'threads':0} try: output = subprocess.check_output('ps o user,pcpu,pmem,etime,nlwp %i' % pid, shell=True) - try: - output = output.decode() - except AttributeError: - # Python2 catch - pass + output = output.decode() output = output.split('\n')[1] fields = output.split(None, 4) data['user'] = fields[0] @@ -96,7 +89,6 @@ def get_command_line(pid): with open('/proc/%i/cmdline' % pid, 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') - fh.close() except IOError: pass return cmd diff --git a/tools/like_top.py b/tools/like_top.py index 4f6d0ccda..56d5e6c87 100755 --- a/tools/like_top.py +++ b/tools/like_top.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,12 +26,6 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Python2 compatibility -from __future__ import print_function -import sys -if sys.version_info < (3,): - range = xrange import os import glob @@ -42,10 +36,7 @@ import traceback import subprocess -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO +from io import StringIO os.environ['VMA_TRACELEVEL'] = '0' from bifrost.proclog import PROCLOG_DIR, load_by_pid @@ -102,8 +93,7 @@ def get_processor_usage(): with open('/proc/stat', 'r') as fh: lines = fh.read() - fh.close() - + for line in lines.split('\n'): if line[:3] == 'cpu': fields = line.split(None, 10) @@ -156,8 +146,7 @@ def get_memory_swap_usage(): with open('/proc/meminfo', 'r') as fh: lines = fh.read() - fh.close() - + for line in lines.split('\n'): fields = line.split(None, 2) if fields[0] == 'MemTotal:': @@ -230,7 +219,6 @@ def get_command_line(pid): with open('/proc/%i/cmdline' % pid, 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') - fh.close() except IOError: pass return cmd diff --git a/tools/pipeline2dot.py b/tools/pipeline2dot.py index b6f20f640..d0996c0ad 100755 --- a/tools/pipeline2dot.py +++ b/tools/pipeline2dot.py @@ -1,7 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import os import sys import glob @@ -94,7 +91,6 @@ def get_command_line(pid): with open('/proc/%i/cmdline' % pid, 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') - fh.close() except IOError: pass return cmd diff --git a/tools/setirq.py b/tools/setirq.py index 116b7c245..f2456fce7 100755 --- a/tools/setirq.py +++ b/tools/setirq.py @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - import argparse from bifrost import telemetry @@ -57,9 +54,8 @@ def write_irq_smp_affinity(irq, mask): def main(args): - fh = open('/proc/interrupts', 'r') - lines = fh.read() - fh.close() + with open('/proc/interrupts', 'r') as fh: + lines = fh.read() irqs = {} for line in lines.split('\n'): From d2e4ece370b1b4d86ebd43c8c5c9ace2a667d3d1 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:02:14 -0700 Subject: [PATCH 08/56] Drop PyPy stuff. --- python/bifrost/pypy3_compat.c | 71 ----------------------------------- python/setup.py | 14 ++----- 2 files changed, 3 insertions(+), 82 deletions(-) delete mode 100644 python/bifrost/pypy3_compat.c diff --git a/python/bifrost/pypy3_compat.c b/python/bifrost/pypy3_compat.c deleted file mode 100644 index 44ad9302a..000000000 --- a/python/bifrost/pypy3_compat.c +++ /dev/null @@ -1,71 +0,0 @@ - -/* - * Copyright (c) 2021, The Bifrost Authors. All rights reserved. - * Copyright (c) 2021, The University of New Mexico. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of The Bifrost Authors nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/*! \file pypy3_compat.c - * \brief Compatibility layer for PyPy3 - */ - -#include "Python.h" -#include - -static PyObject* PyMemoryView_FromAddressAndSize(PyObject *self, PyObject *args, PyObject *kwds) { - PyObject *address, *nbyte, *flags, *view; - if(!PyArg_ParseTuple(args, "OOO", &address, &nbyte, &flags)) { - PyErr_Format(PyExc_RuntimeError, "Invalid parameters"); - return NULL; - } - - long addr, size, flgs; - addr = PyLong_AsLong(address); - size = PyLong_AsLong(nbyte); - flgs = PyLong_AsLong(flags); - - char *buf = (char *) addr; - - view = PyMemoryView_FromMemory(buf, size, flgs | PyBUF_READ); - return view; -} - -static PyMethodDef CompatMethods[] = { - {"PyMemoryView_FromMemory", (PyCFunction) PyMemoryView_FromAddressAndSize, METH_VARARGS, NULL}, - {NULL, NULL, 0, NULL} -}; - -static struct PyModuleDef Compat = { - PyModuleDef_HEAD_INIT, "_pypy3_compat", NULL, -1, CompatMethods,}; - -PyMODINIT_FUNC PyInit__pypy3_compat(void) { - PyObject *m; - m = PyModule_Create(&Compat); - if(m == NULL) { - return NULL; - } - return m; -} diff --git a/python/setup.py b/python/setup.py index b1107a721..623a93cd7 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +#!/usr/bin/env python3 +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,9 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import print_function - from setuptools import setup, Extension, find_packages import os import sys @@ -55,11 +52,6 @@ print("*************************************************************************") raise -# Build the PyPy3 compatibility module, if needed -modules = [] -if sys.version.find('PyPy') != -1: - modules.append(Extension('_pypy3_compat', ['bifrost/pypy3_compat.c'])) - # Build up a list of scripts to install scripts = glob.glob(os.path.join('..', 'tools', '*.py')) @@ -80,4 +72,4 @@ "matplotlib" ], ext_package='bifrost', - ext_modules = modules) + ext_modules = []]) From 323f36fc476bf43d278db614b9fbd75d8cbc43f0 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:09:38 -0700 Subject: [PATCH 09/56] Cleanup. --- tools/getirq.py | 1 - tools/getsiblings.py | 1 - tools/like_bmon.py | 13 ++++++------- tools/like_pmap.py | 5 ++--- tools/like_ps.py | 5 ++--- tools/like_top.py | 5 ++--- tools/pipeline2dot.py | 15 ++++++--------- tools/setirq.py | 3 +-- 8 files changed, 19 insertions(+), 29 deletions(-) diff --git a/tools/getirq.py b/tools/getirq.py index cd426e153..5a77f8f56 100755 --- a/tools/getirq.py +++ b/tools/getirq.py @@ -65,4 +65,3 @@ def main(args): help='interface to query') args = parser.parse_args() main(args) - diff --git a/tools/getsiblings.py b/tools/getsiblings.py index 2994eb81f..ee17c56b3 100755 --- a/tools/getsiblings.py +++ b/tools/getsiblings.py @@ -76,4 +76,3 @@ def main(args): help='core to query') args = parser.parse_args() main(args) - diff --git a/tools/like_bmon.py b/tools/like_bmon.py index e9630278f..1b44f6bfe 100755 --- a/tools/like_bmon.py +++ b/tools/like_bmon.py @@ -79,11 +79,11 @@ def get_transmit_receive(): except KeyError: good, missing, invalid, late, nvalid = 0, 0, 0, 0, 0 - blockList['%i-%s' % (pid, block)] = {'pid': pid, 'name':block, - 'time':t, - 'good': good, 'missing': missing, - 'invalid': invalid, 'late': late, - 'nvalid': nvalid} + blockList[f"{pid}-{block}"] = {'pid': pid, 'name':block, + 'time':t, + 'good': good, 'missing': missing, + 'invalid': invalid, 'late': late, + 'nvalid': nvalid} return blockList @@ -97,7 +97,7 @@ def get_command_line(pid): cmd = '' try: - with open('/proc/%i/cmdline' % pid, 'r') as fh: + with open(f"/proc/{pid}/cmdline", 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') except IOError: @@ -412,4 +412,3 @@ def main(args): ) args = parser.parse_args() main(args) - diff --git a/tools/like_pmap.py b/tools/like_pmap.py index 74d0203fe..f5a2e2ee4 100755 --- a/tools/like_pmap.py +++ b/tools/like_pmap.py @@ -86,8 +86,8 @@ def main(args): # Load in the NUMA map page for this process try: - fh = open('/proc/%i/numa_maps' % args.pid, 'r') - numaInfo = fh.read() + with open(f"/proc/{pid}/numa_maps", 'r') as fh: + numaInfo = fh.read() except IOError: raise RuntimeError("Cannot find NUMA memory info for PID: %i" % args.pid) @@ -262,4 +262,3 @@ def main(args): help='process ID') args = parser.parse_args() main(args) - diff --git a/tools/like_ps.py b/tools/like_ps.py index 2d729db3a..1ae8a8d77 100755 --- a/tools/like_ps.py +++ b/tools/like_ps.py @@ -62,7 +62,7 @@ def get_process_details(pid): data = {'user':'', 'cpu':0.0, 'mem':0.0, 'etime':'00:00', 'threads':0} try: - output = subprocess.check_output('ps o user,pcpu,pmem,etime,nlwp %i' % pid, shell=True) + output = subprocess.check_output(f"ps o user,pcpu,pmem,etime,nlwp {pid}", shell=True) output = output.decode() output = output.split('\n')[1] fields = output.split(None, 4) @@ -86,7 +86,7 @@ def get_command_line(pid): cmd = '' try: - with open('/proc/%i/cmdline' % pid, 'r') as fh: + with open(f"/proc/{pid}/cmdline", 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') except IOError: @@ -198,4 +198,3 @@ def main(args): ) args = parser.parse_args() main(args) - diff --git a/tools/like_top.py b/tools/like_top.py index 56d5e6c87..4998c6011 100755 --- a/tools/like_top.py +++ b/tools/like_top.py @@ -216,7 +216,7 @@ def get_command_line(pid): cmd = '' try: - with open('/proc/%i/cmdline' % pid, 'r') as fh: + with open(f"/proc/{pid}/cmdline", 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') except IOError: @@ -332,7 +332,7 @@ def main(args): except KeyError: ac, pr, re = 0.0, 0.0, 0.0 - blockList['%i-%s' % (pid, block)] = {'pid': pid, 'name':block, 'cmd': cmd, 'core': cr, 'acquire': ac, 'process': pr, 'reserve': re, 'total':ac+pr+re} + blockList[f"{pid}-{block}"] = {'pid': pid, 'name':block, 'cmd': cmd, 'core': cr, 'acquire': ac, 'process': pr, 'reserve': re, 'total':ac+pr+re} ## Sort order = sorted(blockList, key=lambda x: blockList[x][sort_key], reverse=sort_rev) @@ -438,4 +438,3 @@ def main(args): ) args = parser.parse_args() main(args) - diff --git a/tools/pipeline2dot.py b/tools/pipeline2dot.py index d0996c0ad..4c436387d 100755 --- a/tools/pipeline2dot.py +++ b/tools/pipeline2dot.py @@ -62,10 +62,8 @@ def get_process_details(pid): data = {'user':'', 'cpu':0.0, 'mem':0.0, 'etime':'00:00', 'threads':0} try: - output = subprocess.check_output('ps o user,pcpu,pmem,etime,nlwp %i' % pid, shell=True) - if sys.version_info.major > 2 and isinstance(output, bytes): - # decode the output to utf-8 in python 3 - output = output.decode("utf-8") + output = subprocess.check_output(f"ps o user,pcpu,pmem,etime,nlwp {pid}", shell=True) + output = output.decode() output = output.split('\n')[1] fields = output.split(None, 4) data['user'] = fields[0] @@ -88,7 +86,7 @@ def get_command_line(pid): cmd = '' try: - with open('/proc/%i/cmdline' % pid, 'r') as fh: + with open(f"/proc/{pid}/cmdline", 'r') as fh: cmd = fh.read() cmd = cmd.replace('\0', ' ') except IOError: @@ -166,7 +164,7 @@ def get_data_flows(blocks): if blocks[other_block][log]['complex']: bits *= 2 name = 'cplx' if blocks[other_block][log]['complex'] else 'real' - dtype = '%s%i' % (name, bits) + dtype = f"{name}{bits}" except KeyError: pass elif log not in ('in', 'out'): @@ -194,7 +192,7 @@ def get_data_flows(blocks): refCores = [] for i in range(32): try: - refCores.append( blocks[block]['bind']['core%i' % i] ) + refCores.append( blocks[block]['bind'][f"core{i}"] ) except KeyError: break @@ -208,7 +206,7 @@ def get_data_flows(blocks): other_cores = [] for i in range(32): try: - other_cores.append( blocks[other_block]['bind']['core%i' % i] ) + other_cores.append( blocks[other_block]['bind'][f"core{i}"] ) except KeyError: break @@ -349,4 +347,3 @@ def main(args): help='exclude associated blocks') args = parser.parse_args() main(args) - diff --git a/tools/setirq.py b/tools/setirq.py index f2456fce7..254f322e6 100755 --- a/tools/setirq.py +++ b/tools/setirq.py @@ -70,7 +70,7 @@ def main(args): mi = procs.index(mv) irqs[irq] = {'cpu':mi, 'type':type, 'name':name, 'count':mv} - print("Interface: %s" % args.interface) + print(f"Interface: {args.interface}") print("%4s %16s %16s %7s %7s" % ('IRQ', 'Name', 'Type', 'Old CPU', 'New CPU') ) for i,irq in enumerate(sorted(irqs.keys())): oCPU = irqs[irq]['cpu'] @@ -93,4 +93,3 @@ def main(args): help='CPU to bind to') args = parser.parse_args() main(args) - From d8792d6143a0019517076d5c9c6db1701ca36416 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:10:49 -0700 Subject: [PATCH 10/56] Missed one. --- tools/getirq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/getirq.py b/tools/getirq.py index 5a77f8f56..fac1ff8b9 100755 --- a/tools/getirq.py +++ b/tools/getirq.py @@ -50,7 +50,7 @@ def main(args): irqs[irq] = {'cpu':mi, 'type':type, 'name':name, 'count':mv} total = sum([irqs[irq]['count'] for irq in irqs]) - print("Interface: %s" % args.interface) + print(f"Interface: {args.interface}") print("%4s %16s %16s %4s %6s" % ('IRQ', 'Name', 'Type', 'CPU', 'Usage')) for irq in sorted(irqs.keys()): print("%4i %16s %16s %4i %5.1f%%" % (irq, irqs[irq]['name'], irqs[irq]['type'], irqs[irq]['cpu'], 100.0*irqs[irq]['count']/total)) From 3fcd11e3b485e81a652aecb80b50e0f8a8ee22c1 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:11:55 -0700 Subject: [PATCH 11/56] No longer needed. --- testbench/jenkins.sh | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100755 testbench/jenkins.sh diff --git a/testbench/jenkins.sh b/testbench/jenkins.sh deleted file mode 100755 index d1e8924d1..000000000 --- a/testbench/jenkins.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Part 1 - Synthetic data -## Create -python generate_test_data.py -## Use -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_file_read_write.py -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fft.py -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline your_first_block.py - -# Part 2 - Real data -## Download -python download_breakthrough_listen_data.py -y -## Use -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_guppi.py -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_guppi_reader.py -coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fdmt.py ./testdata/pulsars/blc0_guppi_57407_61054_PSR_J1840%2B5640_0004.fil From 14f8cffc2d59d86906c6572bcd57c8e3fdce4144 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:39:17 -0700 Subject: [PATCH 12/56] More cleanup. --- python/bifrost/DataType.py | 10 +++++----- python/bifrost/Space.py | 4 ++-- python/bifrost/address.py | 2 +- python/bifrost/block.py | 8 ++++---- python/bifrost/dtype.py | 15 +++++++-------- python/bifrost/pipeline.py | 26 +++++++++++++------------- python/bifrost/portaudio.py | 2 +- python/bifrost/proclog.py | 2 +- python/bifrost/psrdada.py | 14 +++++++------- python/bifrost/sigproc.py | 8 ++++---- python/bifrost/sigproc2.py | 14 +++++++------- python/bifrost/units.py | 5 ++--- 12 files changed, 54 insertions(+), 56 deletions(-) diff --git a/python/bifrost/DataType.py b/python/bifrost/DataType.py index f4a69b477..367eb004a 100644 --- a/python/bifrost/DataType.py +++ b/python/bifrost/DataType.py @@ -149,10 +149,10 @@ def __init__(self, t=None): self._veclen = t.shape[0] t = t.base else: - raise TypeError("Unsupported Numpy dtype: " + str(t)) + raise TypeError(f"Unsupported Numpy dtype: {t}") self._nbit = t.itemsize * 8 if t.kind not in set(['i', 'u', 'f', 'c', 'V', 'b']): - raise TypeError('Unsupported data type: %s' % str(t)) + raise TypeError(f"Unsupported data type: {t}") if is_vector_structure(t): # Field structure representing vector self._veclen = len(t.names) t = t[0] @@ -167,7 +167,7 @@ def __init__(self, t=None): elif t in [cf16]: self._kind = 'cf' else: - raise TypeError('Unsupported data type: %s' % str(t)) + raise TypeError(f"Unsupported data type: {t}") elif t.kind == 'b': # Note: Represents booleans as uint8 inside Bifrost self._kind = 'u' @@ -193,9 +193,9 @@ def as_numpy_dtype(self): return np.dtype(','.join((str(base),)*self._veclen)) def __str__(self): if self._veclen == 1: - return '%s%i' % (self._kind, self._nbit) + return f"{self._kind}{self._nbit}" else: - return '%s%i[%i]' % (self._kind, self._nbit, self._veclen) + return f"{self._kind}{self._nbit}[{self._veclen}]" @property def is_complex(self): return self._kind[0] == 'c' diff --git a/python/bifrost/Space.py b/python/bifrost/Space.py index 3945e8599..b4cf4516a 100644 --- a/python/bifrost/Space.py +++ b/python/bifrost/Space.py @@ -47,7 +47,7 @@ def __init__(self, s): if isinstance(s, str): if s not in set(['auto', 'system', 'cuda', 'cuda_host', 'cuda_managed']): - raise ValueError('Invalid space: %s' % s) + raise ValueError(f"Invalid space: '{s}'") self._space = s elif isinstance(s, _bf.BFspace) or isinstance(s, int): if s not in SPACEMAP_TO_STR: @@ -55,7 +55,7 @@ def __init__(self, s): ". Valid spaces: " + str(SPACEMAP_TO_STR.keys())) self._space = SPACEMAP_TO_STR[s] else: - raise ValueError('%s is not a space' % s) + raise ValueError(f"'{s}' is not a space") def as_BFspace(self): return SPACEMAP_FROM_STR[self._space] def __str__(self): diff --git a/python/bifrost/address.py b/python/bifrost/address.py index 5b4bff2ff..d7a9ea0ce 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -58,4 +58,4 @@ def address(self): _check(_bf.bfAddressGetString(self.obj, buflen, buf)) return buf.value.decode() def __str__(self): - return "%s:%i" % (self.address, self.port) + return f"{self.address}:{self.port}" diff --git a/python/bifrost/block.py b/python/bifrost/block.py index c92282750..aefdabf34 100644 --- a/python/bifrost/block.py +++ b/python/bifrost/block.py @@ -910,8 +910,8 @@ def __init__(self, function, inputs=1, outputs=1): @param[in] outputs The number of output rings and the number of output numpy arrays from the function.""" super(NumpyBlock, self).__init__() - self.inputs = ['in_%d' % (i + 1) for i in range(inputs)] - self.outputs = ['out_%d' % (i + 1) for i in range(outputs)] + self.inputs = [f"in_{i + 1}" for i in range(inputs)] + self.outputs = [f"out_{i + 1}" for i in range(outputs)] self.ring_names = {} self.create_ring_names() self.function = function @@ -1061,9 +1061,9 @@ def main(self): if self.grab_headers: self.load_user_headers(headers, arrays) - for outspans in self.write(*['out_%d' % (i + 1) for i in range(len(self.ring_names))]): + for outspans in self.write(*[f"out_{i + 1}" for i in range(len(self.ring_names))]): for i in range(len(self.ring_names)): - dtype = self.header['out_%d' % (i + 1)]['dtype'] + dtype = self.header[f"out_{i + 1}"]['dtype'] outspans[i][:] = arrays[i].astype(np.dtype(dtype).type).ravel() try: diff --git a/python/bifrost/dtype.py b/python/bifrost/dtype.py index 93de54d86..66300c4b2 100644 --- a/python/bifrost/dtype.py +++ b/python/bifrost/dtype.py @@ -89,7 +89,7 @@ def numpy2bifrost(dtype): elif dtype == np.complex128: return _bf.BF_DTYPE_CF64 elif dtype == np.complex256 \ and BF_FLOAT128_ENABLED: return _bf.BF_DTYPE_CF128 - else: raise ValueError("Unsupported dtype: " + str(dtype)) + else: raise ValueError(f"Unsupported dtype: {dtype}") def name_nbit2numpy(name, nbit): if name == 'i': @@ -97,19 +97,19 @@ def name_nbit2numpy(name, nbit): elif nbit == 16: return np.int16 elif nbit == 32: return np.int32 elif nbit == 64: return np.int64 - else: raise TypeError("Invalid signed integer type size: %i" % nbit) + else: raise TypeError(f"Invalid signed integer type size: {nbit}") elif name == 'u': if nbit == 8: return np.uint8 elif nbit == 16: return np.uint16 elif nbit == 32: return np.uint32 elif nbit == 64: return np.uint64 - else: raise TypeError("Invalid unsigned integer type size: %i" % nbit) + else: raise TypeError(f"Invalid unsigned integer type size: {nbit}") elif name == 'f': if nbit == 16: return np.float16 elif nbit == 32: return np.float32 elif nbit == 64: return np.float64 elif nbit == 128: return np.float128 - else: raise TypeError("Invalid floating-point type size: %i" % nbit) + else: raise TypeError(f"Invalid floating-point type size: {nbit}") elif name == 'ci': if nbit == 8: return ci8 elif nbit == 16: return ci16 @@ -122,10 +122,9 @@ def name_nbit2numpy(name, nbit): elif nbit == 32: return np.complex64 elif nbit == 64: return np.complex128 elif nbit == 128: return np.complex256 - else: raise TypeError("Invalid complex floating-point type size: %i" % - nbit) + else: raise TypeError(f"Invalid complex floating-point type size: {nbit}") else: - raise TypeError("Invalid type name: " + name) + raise TypeError(f"Invalid type name: {name}") def string2numpy(dtype_str): return name_nbit2numpy(*split_name_nbit(dtype_str)) @@ -145,7 +144,7 @@ def numpy2string(dtype): elif dtype == np.complex64: return 'cf32' elif dtype == np.complex128: return 'cf64' elif dtype == np.complex256: return 'cf128' - else: raise TypeError("Unsupported dtype: " + str(dtype)) + else: raise TypeError(f"Unsupported dtype: {dtype}") def bifrost2string(dtype): """ Convert bifrost BF_DTYPE integer code to ndarray string """ diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index adab8a71d..997a28048 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -90,7 +90,7 @@ def __init__(self, share_temp_storage=False, fuse=False): if name is None: - name = 'BlockScope_%i' % BlockScope.instance_count + name = f"BlockScope_{BlockScope.instance_count}" BlockScope.instance_count += 1 self._name = name self._gulp_nframe = gulp_nframe @@ -120,7 +120,7 @@ def __exit__(self, type, value, tb): def __getattr__(self, name): # Use child's value if set, othersize defer to parent if '_'+name not in self.__dict__: - raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__, name)) + raise AttributeError(f"'{self.__class__}' object has no attribute '{name}'") self_value = getattr(self, '_' + name) if self_value is not None: return self_value @@ -163,9 +163,9 @@ def dot_graph(self, parent_graph=None): #graph_attr = {'label': self._name} graph_attr = {} if parent_graph is None: - g = Digraph('cluster_' + self._name, graph_attr=graph_attr) + g = Digraph(f"cluster_{self._name}", graph_attr=graph_attr) else: - g = parent_graph.subgraph('cluster_' + self._name, + g = parent_graph.subgraph(f"cluster_{self.name}", label=self._name) for child in self._children: if isinstance(child, Block): @@ -221,7 +221,7 @@ class Pipeline(BlockScope): instance_count = 0 def __init__(self, name=None, **kwargs): if name is None: - name = 'Pipeline_%i' % Pipeline.instance_count + name = f"Pipeline_{Pipeline.instance_count}" Pipeline.instance_count += 1 super(Pipeline, self).__init__(name=name, **kwargs) self.blocks = [] @@ -240,7 +240,7 @@ def synchronize_block_initializations(self): if not init_succeeded: self.shutdown() raise PipelineInitError( - "The following block failed to initialize: " + block.name) + f"The following block failed to initialize: {block.name}") # Tell blocks that they can begin data processing self.all_blocks_finished_initializing_event.set() def run(self): @@ -264,7 +264,7 @@ def shutdown(self): join_all(self.threads, timeout=self.shutdown_timeout) for thread in self.threads: if thread.is_alive(): - warnings.warn("Thread %s did not shut down on time and will be killed" % thread.name, RuntimeWarning) + warnings.warn(f"Thread {thread.name} did not shut down on time and will be killed", RuntimeWarning) def shutdown_on_signals(self, signals=None): if signals is None: signals = [signal.SIGHUP, @@ -325,7 +325,7 @@ def __init__(self, irings, type_=None, **kwargs): self.type = type_ or self.__class__.__name__ - self.name = name or '%s_%i' % (self.type, Block.instance_counts[self.type]) + self.name = name or f"{self.type}_{Block.instance_counts[self.type]}" Block.instance_counts[self.type] += 1 super(Block, self).__init__(**kwargs) self.pipeline = get_default_pipeline() @@ -346,7 +346,7 @@ def __init__(self, irings, rnames = {'nring': len(self.irings)} for i, r in enumerate(self.irings): - rnames['ring%i' % i] = r.name + rnames[f"ring{i}"] = r.name self.in_proclog.update(rnames) self.init_trace = ''.join(traceback.format_stack()[:-1]) def shutdown(self): @@ -445,7 +445,7 @@ def __init__(self, sourcenames, gulp_nframe, space=None, *args, **kwargs): rnames = {'nring': len(self.orings)} for i, r in enumerate(self.orings): - rnames['ring%i' % i] = r.name + rnames[f"ring{i}"] = r.name self.out_proclog.update(rnames) def main(self, orings): @@ -458,7 +458,7 @@ def main(self, orings): if 'time_tag' not in ohdr: ohdr['time_tag'] = self._seq_count if 'name' not in ohdr: - ohdr['name'] = 'unnamed-sequence-%i' % self._seq_count + ohdr['name'] = f"unnamed-sequence-{self._seq_count}" self._seq_count += 1 with ExitStack() as oseq_stack: oseqs, ogulp_overlaps = self.begin_sequences( @@ -522,13 +522,13 @@ def __init__(self, irings_, guarantee=True, *args, **kwargs): for iring in self.irings] self._seq_count = 0 self.perf_proclog = ProcLog(self.name + "/perf") - self.sequence_proclogs = [ProcLog(self.name + "/sequence%i" % i) + self.sequence_proclogs = [ProcLog(self.name + f"/sequence{i}") for i in range(len(self.irings))] self.out_proclog = ProcLog(self.name + "/out") rnames = {'nring': len(self.orings)} for i, r in enumerate(self.orings): - rnames['ring%i' % i] = r.name + rnames[f"ring{i}"] = r.name self.out_proclog.update(rnames) def main(self, orings): diff --git a/python/bifrost/portaudio.py b/python/bifrost/portaudio.py index 89f5a612e..0366e8b38 100644 --- a/python/bifrost/portaudio.py +++ b/python/bifrost/portaudio.py @@ -239,7 +239,7 @@ def get_device_count(): if __name__ == "__main__": import portaudio as audio import numpy as np - print("Found %i audio devices" % audio.get_device_count()) + print(f"Found {audio.get_device_count()} audio devices") with audio.open(nbits=16) as audio_stream: nframe = 20 print(repr(audio_stream.read(nframe).raw)) diff --git a/python/bifrost/proclog.py b/python/bifrost/proclog.py index 46a7834fc..cabe4c1f1 100644 --- a/python/bifrost/proclog.py +++ b/python/bifrost/proclog.py @@ -111,7 +111,7 @@ def load_by_pid(pid, include_rings=False): # Make sure we have a directory to load from baseDir = os.path.join(PROCLOG_DIR, str(pid)) if not os.path.isdir(baseDir): - raise RuntimeError("Cannot find log directory associated with PID %s" % pid) + raise RuntimeError(f"Cannot find log directory associated with PID {pid}") # Load contents = {} diff --git a/python/bifrost/psrdada.py b/python/bifrost/psrdada.py index 7b946e495..1e6eb6d50 100644 --- a/python/bifrost/psrdada.py +++ b/python/bifrost/psrdada.py @@ -56,7 +56,7 @@ class MultiLog(object): count = 0 def __init__(self, name=None): if name is None: - name = "MultiLog%i" % MultiLog.count + name = f"MultiLog{MultiLog.count}" MultiLog.count += 1 self.obj = _dada.multilog_open(name, '\0') def __del__(self): @@ -211,25 +211,25 @@ def _connect(self, buffer_key=0xDADA): self.buffer_key = buffer_key _dada.dada_hdu_set_key(self.hdu, self.buffer_key) if _dada.dada_hdu_connect(self.hdu) < 0: - raise IOError("Could not connect to buffer '%x'" % self.buffer_key) + raise IOError(f"Could not connect to buffer '{self.buffer_key:x}'") def _disconnect(self): if _dada.dada_hdu_disconnect(self.hdu) < 0: - raise IOError("Could not disconnect from buffer '%x'" % self.buffer_key) + raise IOError(f"Could not disconnect from buffer '{self.buffer_key:x}'") def _lock(self, mode): self.mode = mode if mode == 'read': if _dada.dada_hdu_lock_read(self.hdu) < 0: - raise IOError("Could not lock buffer '%x' for reading" % self.buffer_key) + raise IOError(f"Could not lock buffer '{self.buffer_key:x}' for reading") else: if _dada.dada_hdu_lock_write(self.hdu) < 0: - raise IOError("Could not lock buffer '%x' for writing" % self.buffer_key) + raise IOError(f"Could not lock buffer '{self.buffer_key:x}' for writing") def _unlock(self): if self.mode == 'read': if _dada.dada_hdu_unlock_read(self.hdu) < 0: - raise IOError("Could not unlock buffer '%x' for reading" % self.buffer_key) + raise IOError(f"Could not unlock buffer '{self.buffer_key:x}' for reading") else: if _dada.dada_hdu_unlock_write(self.hdu) < 0: - raise IOError("Could not unlock buffer '%x' for writing" % self.buffer_key) + raise IOError(f"Could not unlock buffer '{self.buffer_key:x}' for writing") def relock(self): self._unlock() self._lock(self.mode) diff --git a/python/bifrost/sigproc.py b/python/bifrost/sigproc.py index 8f4a6f145..2daf52c46 100644 --- a/python/bifrost/sigproc.py +++ b/python/bifrost/sigproc.py @@ -169,8 +169,8 @@ def _write_header(hdr, file_object): elif key == "header_size": pass else: - #raise KeyError("Unknown sigproc header key: %s"%key) - warnings.warn("Unknown sigproc header key: '%s'" % key, RuntimeWarning) + #raise KeyError(f"Unknown sigproc header key: {key}") + warnings.warn(f"Unknown sigproc header key: '{key}'", RuntimeWarning) _header_write_string(file_object, "HEADER_END") def _read_header(file_object): @@ -199,7 +199,7 @@ def _read_header(file_object): header[expecting] = key expecting = None else: - warnings.warn("Unknown header key: '%s'" % key, RuntimeWarning) + warnings.warn(f"Unknown header key: '{key}'", RuntimeWarning) if 'nchans' not in header: header['nchans'] = 1 header['header_size'] = file_object.tell() @@ -231,7 +231,7 @@ def seek_to_data(file_object): header[expecting] = key expecting = None else: - warnings.warn("Unknown header key: '%s'" % key, RuntimeWarning) + warnings.warn(f"Unknown header key: '{key}'", RuntimeWarning) return def pack(data, nbit): diff --git a/python/bifrost/sigproc2.py b/python/bifrost/sigproc2.py index 4f1e6d4a3..c21b22f23 100644 --- a/python/bifrost/sigproc2.py +++ b/python/bifrost/sigproc2.py @@ -183,8 +183,8 @@ def write_header(hdr, f): elif key in _character_values: _header_write(f, key, int(val), fmt='=b') else: - #raise KeyError("Unknown sigproc header key: %s"%key) - warnings.warn("Unknown sigproc header key: '%s'" % key, RuntimeWarning) + #raise KeyError(f"Unknown sigproc header key: {key}") + warnings.warn(f"Unknown sigproc header key: '{key}'", RuntimeWarning) _header_write_string(f, "HEADER_END") def _read_header(f): @@ -211,7 +211,7 @@ def _read_header(f): header[expecting] = key expecting = None else: - warnings.warn("Unknown header key: '%s'" % key, RuntimeWarning) + warnings.warn(f"Unknown header key: '{key}'", RuntimeWarning) if 'nchans' not in header: header['nchans'] = 1 header['header_size'] = f.tell() @@ -252,7 +252,7 @@ def unpack(data, nbit): x = x << 7 # Shift into high bits to induce sign-extension return x.view(data.dtype) >> 7 else: - raise ValueError("unpack: unexpected nbit! (%i)" % nbit) + raise ValueError(f"unpack: unexpected nbit! ({nbit})") # TODO: Add support for writing # Add support for data_type != filterbank @@ -374,11 +374,11 @@ def readinto(self, buf): def __str__(self): hmod = self.header.copy() d = hmod['data_type'] - hmod['data_type'] = "%i (%s)" % (d, _data_types[d]) + hmod['data_type'] = f"{d} ({_data_types[d]})" t = hmod['telescope_id'] - hmod['telescope_id'] = "%i (%s)" % (t, _telescopes[t]) + hmod['telescope_id'] = f"{t} ({_telescopes[t]})" m = hmod['machine_id'] - hmod['machine_id'] = "%i (%s)" % (m, _machines[m]) + hmod['machine_id'] = f"{m} ({_machines[m]})" return '\n'.join(['% 16s: %s' % (key, val) for (key, val) in hmod.items()]) def __getitem__(self, key): diff --git a/python/bifrost/units.py b/python/bifrost/units.py index e9b6e81d6..5b38ba3cd 100644 --- a/python/bifrost/units.py +++ b/python/bifrost/units.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -37,8 +37,7 @@ def convert_units(value, old_units, new_units): try: new_quantity = old_quantity.to(new_units) except pint.DimensionalityError: - raise ValueError("Cannot convert units %s to %s" % - (old_units, new_units)) + raise ValueError(f"Cannot convert units {old_units} to {new_units}") return new_quantity.magnitude # TODO: May need something more flexible, like a Units wrapper class with __str__ From 255ace319b8b88bd656f0d32267cce0fc40120d5 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 19:41:05 -0700 Subject: [PATCH 13/56] We still need sys. --- python/bifrost/ndarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/bifrost/ndarray.py b/python/bifrost/ndarray.py index 2b45f2352..d632b0897 100644 --- a/python/bifrost/ndarray.py +++ b/python/bifrost/ndarray.py @@ -36,6 +36,7 @@ """ +import sys import ctypes import numpy as np from bifrost.memory import raw_malloc, raw_free, raw_get_space, space_accessible From b517c62d7405d0df8845843a5af8e3452a0fae44 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 20:15:51 -0700 Subject: [PATCH 14/56] Ugh. --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 623a93cd7..3c0ba4c22 100755 --- a/python/setup.py +++ b/python/setup.py @@ -72,4 +72,4 @@ "matplotlib" ], ext_package='bifrost', - ext_modules = []]) + ext_modules = []) From 20b399e4bab9b7a327136ff90ab3be3885d04ea4 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 20:19:23 -0700 Subject: [PATCH 15/56] We still need sys. --- python/bifrost/pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index 997a28048..57c410e51 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -25,11 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import sys import threading -try: - import queue -except ImportError: - import Queue as queue +import queue import time import signal import warnings From a9b74f01197bec415e8fff6a3610f601ab597ca5 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 13 Feb 2023 20:23:12 -0700 Subject: [PATCH 16/56] Selective decode. --- python/bifrost/ring2.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/python/bifrost/ring2.py b/python/bifrost/ring2.py index 06ef49ac3..a82f8cb15 100644 --- a/python/bifrost/ring2.py +++ b/python/bifrost/ring2.py @@ -173,11 +173,7 @@ def ring(self): @property def name(self): n = _get(_bf.bfRingSequenceGetName, self._base_obj) - try: - n = n.decode() - except AttributeError: - pass - return n + return n.decode() @property def time_tag(self): return _get(_bf.bfRingSequenceGetTimeTag, self._base_obj) @@ -200,7 +196,6 @@ def tensor(self): # TODO: This shouldn't be public nringlet = reduce(lambda x, y: x * y, ringlet_shape, 1) frame_nelement = reduce(lambda x, y: x * y, frame_shape, 1) dtype = header['_tensor']['dtype'] - dtype = dtype.decode() nbit = DataType(dtype).itemsize_bits assert(nbit % 8 == 0) frame_nbyte = frame_nelement * nbit // 8 @@ -239,7 +234,7 @@ def __init__(self, ring, header, gulp_nframe, buf_nframe): tensor = self.tensor # **TODO: Consider moving this into bfRingSequenceBegin self.ring.resize(gulp_nframe * tensor['frame_nbyte'], - buf_nframe * tensor['frame_nbyte'], + buf_nframe * tensor['frame_nbyte'], tensor['nringlet']) offset_from_head = 0 # TODO: How to allow time_tag to be optional? Probably need to plumb support through to backend. From be95d0034b1e1217bbb2f156e76ef8240511c26b Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 07:22:36 -0700 Subject: [PATCH 17/56] Missed a few more. --- python/bifrost/telemetry/__init__.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/python/bifrost/telemetry/__init__.py b/python/bifrost/telemetry/__init__.py index 0b22e6ef3..fd6e8ac26 100644 --- a/python/bifrost/telemetry/__init__.py +++ b/python/bifrost/telemetry/__init__.py @@ -33,12 +33,8 @@ import socket import inspect import warnings -try: - from urllib2 import urlopen - from urllib import urlencode -except ImportError: - from urllib.request import urlopen - from urllib.parse import urlencode +from urllib.request import urlopen +from urllib.parse import urlencode from threading import RLock from functools import wraps @@ -153,11 +149,7 @@ def send(self, final=False): 'version' : self.version, 'session_time': "%.6f" % ((tNow-self._session_start) if final else 0.0,), 'payload' : payload}) - try: - payload = payload.encode() - except AttributeError: - pass - uh = urlopen('https://fornax.phys.unm.edu/telemetry/bifrost.php', payload, + uh = urlopen('https://fornax.phys.unm.edu/telemetry/bifrost.php', payload.encode(), timeout=TELEMETRY_TIMEOUT) status = uh.read() if status == '': From 1f06f6c186cdcab3736294e31cb593610ce71754 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 08:22:19 -0700 Subject: [PATCH 18/56] Try out type hinting. --- python/bifrost/address.py | 15 +++++++++------ python/bifrost/affinity.py | 10 ++++++---- python/bifrost/libbifrost.py | 12 ++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/python/bifrost/address.py b/python/bifrost/address.py index d7a9ea0ce..ff97f0718 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -29,12 +29,15 @@ import ctypes from socket import AF_UNSPEC +from typing import NewVar, Optional from bifrost import telemetry telemetry.track_module() +AFfamily = NewVar('AFfamily', int) + class Address(BifrostObject): - def __init__(self, address, port, family=None): + def __init__(self, address: str, port: int, family: Optional[AFfamily]=None): address = address.encode() assert(isinstance(port, int)) if family is None: @@ -43,19 +46,19 @@ def __init__(self, address, port, family=None): self, _bf.bfAddressCreate, _bf.bfAddressDestroy, address, port, family) @property - def family(self): + def family(self) -> int: return _get(_bf.bfAddressGetFamily, self.obj) @property - def port(self): + def port(self) -> int: return _get(_bf.bfAddressGetPort, self.obj) @property - def mtu(self): + def mtu(self) -> int: return _get(_bf.bfAddressGetMTU, self.obj) @property - def address(self): + def address(self) -> str: buflen = 128 buf = ctypes.create_string_buffer(buflen) _check(_bf.bfAddressGetString(self.obj, buflen, buf)) return buf.value.decode() - def __str__(self): + def __str__(self) -> str: return f"{self.address}:{self.port}" diff --git a/python/bifrost/affinity.py b/python/bifrost/affinity.py index 854db90f3..c85f97ccd 100644 --- a/python/bifrost/affinity.py +++ b/python/bifrost/affinity.py @@ -26,16 +26,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check, _get, _array +from bifrost.libbifrost import _bf, _check, _get, _array, BFstatusT + +from typing import List from bifrost import telemetry telemetry.track_module() -def get_core(): +def get_core() -> int: return _get(_bf.bfAffinityGetCore) -def set_core(core): +def set_core(core: int) -> BFstatusT: _check(_bf.bfAffinitySetCore(core)) -def set_openmp_cores(cores): +def set_openmp_cores(cores: List[int]) -> BFstatusT: # PYCLIBRARY ISSUE # TODO: Would be really nice to be able to directly pass # a list here instead of needing to specify _array+type. diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index ca643688c..af03479f6 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -35,12 +35,24 @@ # E.g., _bf.bfRingSequenceGetName() [should be ] import ctypes +from typing import NewType import bifrost.libbifrost_generated as _bf bf = _bf # Public access to library from bifrost import telemetry telemetry.track_module() +# Typing helpers + +#: Type for Bifrost return codes +BFstatusT = NewType('BFstatus', int) + +#: Type for Bifrost memory spaces +BFspaceT = NewType('BFspace', int) + +#: Type for Bifrost data types +BFdtypeT = NewType('BFdtype', int) + # Internal helpers below class EndOfDataStop(RuntimeError): From 0bed5c2a94b39451be16ffb1ba905abc817c3dca Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 08:26:52 -0700 Subject: [PATCH 19/56] Var -> Type --- python/bifrost/address.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/address.py b/python/bifrost/address.py index ff97f0718..9da50a3b3 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -29,12 +29,12 @@ import ctypes from socket import AF_UNSPEC -from typing import NewVar, Optional +from typing import NewType, Optional from bifrost import telemetry telemetry.track_module() -AFfamily = NewVar('AFfamily', int) +AFfamily = NewType('AFfamily', int) class Address(BifrostObject): def __init__(self, address: str, port: int, family: Optional[AFfamily]=None): From 58ffa930d6a82981a672eab86f48133fa89230c5 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 08:31:14 -0700 Subject: [PATCH 20/56] Naming cleanup. --- python/bifrost/address.py | 4 ++-- python/bifrost/affinity.py | 4 ++-- python/bifrost/libbifrost.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/bifrost/address.py b/python/bifrost/address.py index 9da50a3b3..408f15069 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -34,10 +34,10 @@ from bifrost import telemetry telemetry.track_module() -AFfamily = NewType('AFfamily', int) +AFfamilyT = NewType('AFfamilyT', int) class Address(BifrostObject): - def __init__(self, address: str, port: int, family: Optional[AFfamily]=None): + def __init__(self, address: str, port: int, family: Optional[AFfamilyT]=None): address = address.encode() assert(isinstance(port, int)) if family is None: diff --git a/python/bifrost/affinity.py b/python/bifrost/affinity.py index c85f97ccd..2dc5973ec 100644 --- a/python/bifrost/affinity.py +++ b/python/bifrost/affinity.py @@ -35,9 +35,9 @@ def get_core() -> int: return _get(_bf.bfAffinityGetCore) -def set_core(core: int) -> BFstatusT: +def set_core(core: int) -> None: _check(_bf.bfAffinitySetCore(core)) -def set_openmp_cores(cores: List[int]) -> BFstatusT: +def set_openmp_cores(cores: List[int]) -> None: # PYCLIBRARY ISSUE # TODO: Would be really nice to be able to directly pass # a list here instead of needing to specify _array+type. diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index af03479f6..4d3824101 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -45,13 +45,13 @@ # Typing helpers #: Type for Bifrost return codes -BFstatusT = NewType('BFstatus', int) +BFstatusT = NewType('BFstatusT', int) #: Type for Bifrost memory spaces -BFspaceT = NewType('BFspace', int) +BFspaceT = NewType('BFspaceT', int) #: Type for Bifrost data types -BFdtypeT = NewType('BFdtype', int) +BFdtypeT = NewType('BFdtypeT', int) # Internal helpers below From 9a6ccbcef0247c129f691b639bbd9ec6afa4d59c Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 11:34:23 -0700 Subject: [PATCH 21/56] Another naming scheme for the type hinting. --- python/bifrost/address.py | 4 ++-- python/bifrost/affinity.py | 2 +- python/bifrost/libbifrost.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/bifrost/address.py b/python/bifrost/address.py index 408f15069..042386d0c 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -34,10 +34,10 @@ from bifrost import telemetry telemetry.track_module() -AFfamilyT = NewType('AFfamilyT', int) +AFfamily_enum = NewType('AFfamily_enum', int) class Address(BifrostObject): - def __init__(self, address: str, port: int, family: Optional[AFfamilyT]=None): + def __init__(self, address: str, port: int, family: Optional[AFfamily_enum]=None): address = address.encode() assert(isinstance(port, int)) if family is None: diff --git a/python/bifrost/affinity.py b/python/bifrost/affinity.py index 2dc5973ec..5f5cedc77 100644 --- a/python/bifrost/affinity.py +++ b/python/bifrost/affinity.py @@ -26,7 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check, _get, _array, BFstatusT +from bifrost.libbifrost import _bf, _check, _get, _array from typing import List diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index 4d3824101..901634f29 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -45,13 +45,13 @@ # Typing helpers #: Type for Bifrost return codes -BFstatusT = NewType('BFstatusT', int) +BFstatus_enum = NewType('BFstatus_enum', int) #: Type for Bifrost memory spaces -BFspaceT = NewType('BFspaceT', int) +BFspace_enum = NewType('BFspace_enum', int) #: Type for Bifrost data types -BFdtypeT = NewType('BFdtypeT', int) +BFdtype_enum = NewType('BFdtype_enum', int) # Internal helpers below From bbd50d99eca9c0adbe315c6235c84bd2700acc37 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 14:37:15 -0700 Subject: [PATCH 22/56] Removed test_fft.cu. --- src/test_fft.cu | 239 ------------------------------------------------ 1 file changed, 239 deletions(-) delete mode 100644 src/test_fft.cu diff --git a/src/test_fft.cu b/src/test_fft.cu deleted file mode 100644 index 8f2b55f7d..000000000 --- a/src/test_fft.cu +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2016, The Bifrost Authors. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of The Bifrost Authors nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/*! \file testfft.cu - * \brief This file tests the fft.cu functionality. - */ - -#include "fft.cu" -#include -#include - -/*! \brief Simple test of 2 dimensional real data - * through a forward bfFFT. - */ -TEST(FFTTest, Handles2dReal) -{ - /// Set up the test with a 3x2 element real array - BFarray dataAndDescription; - BFarray outputDataAndDescription; - cufftReal hostData[3][2] = - {{1,2},{2,3},{3,4}}; - cufftReal** deviceData; - cufftComplex outputHostData[3][2] = {}; - cufftComplex** outputDeviceData; - /// Load the test array to the GPU - cudaMalloc((void**)&deviceData, sizeof(cufftReal)*6); - cudaMalloc((void**)&outputDeviceData, sizeof(cufftComplex)*6); - cudaMemcpy( - deviceData, hostData, - sizeof(cufftReal)*6, cudaMemcpyHostToDevice); - /// Describe our input and output data - dataAndDescription.data = deviceData; - dataAndDescription.space = BF_SPACE_CUDA; - dataAndDescription.shape[0] = 3; - dataAndDescription.shape[1] = 2; - dataAndDescription.dtype = 0; - dataAndDescription.ndim = 2; - dataAndDescription.strides[0] = 2*sizeof(cufftReal); - dataAndDescription.strides[1] = sizeof(cufftReal); - outputDataAndDescription = dataAndDescription; - outputDataAndDescription.data = outputDeviceData; - outputDataAndDescription.dtype = 1; - outputDataAndDescription.strides[0] = 2*sizeof(cufftComplex); - outputDataAndDescription.strides[1] = sizeof(cufftComplex); - - /// Perform the forward FFT in the separate output object - bfFFT(&dataAndDescription, &outputDataAndDescription, FFT_FORWARD); - /// Download the data back from the GPU - cudaMemcpy( - outputHostData, (cufftComplex*)outputDataAndDescription.data, - sizeof(cufftComplex)*6, cudaMemcpyDeviceToHost); - - /// Did the FFT produce the correct result? - /// We check this only with one value. - EXPECT_EQ(cuCrealf(outputHostData[0][0]),15); -} - -/*! \brief Simple test of 1 dimensional real data - * through a forward bfFFT. - */ -TEST(FFTTest, Handles1dReal) -{ - /// Set up the test with a 4 element real array - BFarray dataAndDescription; - BFarray outputDataAndDescription; - cufftReal hostData[4] = {1,3,6,4.5}; - cufftReal* deviceData; - cufftComplex outputHostData[3]; - cufftComplex* outputDeviceData; - /// Load this array to the GPU - cudaMalloc((void**)&deviceData, sizeof(cufftReal)*5); - cudaMalloc((void**)&outputDeviceData, sizeof(cufftComplex)*3); - cudaMemcpy( - deviceData, hostData, - sizeof(cufftReal)*4, cudaMemcpyHostToDevice); - /// Describe our input and output data - dataAndDescription.data = deviceData; - dataAndDescription.space = BF_SPACE_CUDA; - dataAndDescription.shape[0] = 4; - dataAndDescription.dtype = 0; - dataAndDescription.ndim = 1; - dataAndDescription.strides[0] = sizeof(cufftReal); - /// Output data is a different type, so requires - /// a separate output object - outputDataAndDescription = dataAndDescription; - outputDataAndDescription.data = outputDeviceData; - outputDataAndDescription.dtype = 1; - outputDataAndDescription.strides[0] = sizeof(cufftComplex); - - /// Perform the forward FFT into the separate output data - bfFFT(&dataAndDescription, &outputDataAndDescription, FFT_FORWARD); - /// Download the data back from the GPU - cudaMemcpy( - outputHostData, (cufftComplex*)outputDataAndDescription.data, - sizeof(cufftComplex)*3, cudaMemcpyDeviceToHost); - - /// Did the FFT produce the correct result? - /// We check this only with one value. - EXPECT_EQ((int)cuCimagf(outputHostData[1]), 1); -} - -/*! \brief Simple test of 2 dimensional complex data - * through a forward bfFFT. - */ -TEST(FFTTest, Handles2dComplex) -{ - /// Set up the test with a 3x3 element complex array - BFarray dataAndDescription; - cufftComplex hostData[3][3] = - {{{5,1},{0,0},{100,0}}, - {{5,1},{30,0},{100,0}}, - {{30,0},{0,0},{10,1}}}; - /// Load this array to the GPU - cufftComplex** deviceData; - cudaMalloc((void**)&deviceData, sizeof(cufftComplex)*9); - cudaMemcpy( - deviceData, hostData, - sizeof(cufftComplex)*9, cudaMemcpyHostToDevice); - dataAndDescription.data = deviceData; - /// Describe our data - dataAndDescription.space = BF_SPACE_CUDA; - dataAndDescription.shape[0] = 3; - dataAndDescription.shape[1] = 3; - dataAndDescription.dtype = 1; - dataAndDescription.ndim = 2; - dataAndDescription.strides[0] = 3*sizeof(cufftComplex); - dataAndDescription.strides[1] = sizeof(cufftComplex); - - /// Perform the forward FFT in place - bfFFT(&dataAndDescription, &dataAndDescription, FFT_FORWARD); - /// Dowload data back from the GPU - cudaMemcpy( - hostData, (cufftComplex**)dataAndDescription.data, - sizeof(cufftComplex)*9, cudaMemcpyDeviceToHost); - - /// Did the FFT produce the correct result? - /// We check this only with one value. - EXPECT_EQ((int)cuCimagf(hostData[2][2]), -125); -} - -/*! \brief Simple test of 1 dimensional complex data - * through an inverse bfFFT. - */ -TEST(FFTTest, InverseC2C) -{ - /// Set up the test with a 5 element complex array - BFarray dataAndDescription; - cufftComplex hostData[5] = {{0,-1},{0,0},{100,-100},{30,0},{-5,0}}; - /// Load this array to the GPU - cufftComplex* deviceData; - cudaMalloc((void**)&deviceData, sizeof(cufftComplex)*5); - cudaMemcpy(deviceData, hostData, sizeof(cufftComplex)*5, cudaMemcpyHostToDevice); - dataAndDescription.data = deviceData; - /// Describe our data - dataAndDescription.space = BF_SPACE_CUDA; - dataAndDescription.shape[0] = 5; - dataAndDescription.dtype = 1; - dataAndDescription.ndim = 1; - dataAndDescription.strides[0] = sizeof(cufftComplex); - - /// Perform the inverse FFT in place - bfFFT(&dataAndDescription, &dataAndDescription, FFT_INVERSE); - /// Download back from the GPU - cudaMemcpy( - hostData, (cufftComplex*)dataAndDescription.data, - sizeof(cufftComplex)*5, cudaMemcpyDeviceToHost); - - /// Did the FFT produce the correct answer? - EXPECT_EQ((int)cuCrealf(hostData[3]),139); -} - -// TODO: Make this data actually complex aside from type -/*! \brief Simple test of 1 dimensional complex data - * through a forward bfFFT. - */ -TEST(FFTTest, Handles1dComplex) -{ - /// Set up the test with a 5 element complex array - BFarray dataAndDescription; - cufftComplex hostData[5] = - {{0,0},{30,0},{100,0},{30,0},{-5,0}}; - /// Load this array to the GPU - cufftComplex* deviceData; - cudaMalloc((void**)&deviceData, sizeof(cufftComplex)*5); - cudaMemcpy(deviceData, hostData, sizeof(cufftComplex)*5, cudaMemcpyHostToDevice); - dataAndDescription.data = deviceData; - /// Describe our data - dataAndDescription.space = BF_SPACE_CUDA; - dataAndDescription.shape[0] = 5; - dataAndDescription.dtype = 1; - dataAndDescription.ndim = 1; - dataAndDescription.strides[0] = sizeof(cufftComplex); - - /// Perform the forward FFT in place - bfFFT(&dataAndDescription, &dataAndDescription, FFT_FORWARD); - /// Download back from the GPU. - cudaMemcpy( - hostData, (cufftComplex*)dataAndDescription.data, - sizeof(cufftComplex)*5, cudaMemcpyDeviceToHost); - - /// Test a single value of the output, to test for - /// the FFT's correct answer? - EXPECT_EQ((int)cuCimagf(hostData[4]),(int)74.431946); -} - -// TODO: Add task functionality for rings? -// TODO: Add test for multiple GPUs. -// TODO: Add test for type conversion. - -int main(int argc, char** argv) -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} From eadb105ab0b8dd916108a3fd9fb9cc0cf6d91f8c Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 18:04:44 -0700 Subject: [PATCH 23/56] Yet another approach to type hinting. --- python/Makefile.in | 6 ++++++ python/bifrost/address.py | 8 +++----- python/bifrost/libbifrost.py | 15 ++------------ python/typehinting.py | 40 ++++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 python/typehinting.py diff --git a/python/Makefile.in b/python/Makefile.in index ac0d13877..4ee2c30de 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -34,6 +34,11 @@ define run_ctypesgen @SED@ -i.orig -e '/errcheck = ReturnString/s/^/#/' $@ endef +define run_typehinting + # Build the libbifrost typing hinting + @PYTHON@ -c 'from typehinting import build_typehinting; build_typehinting("$@")' +endef + ifeq "$(wildcard $(PSRDADA_HEADERS))" "" PSRDADA_PYTHON_BINDINGS_FILE = endif @@ -46,6 +51,7 @@ $(PSRDADA_PYTHON_BINDINGS_FILE): $(PSRDADA_HEADERS) $(BIFROST_PYTHON_BINDINGS_FILE): $(INC_DIR)/bifrost/*.h $(call run_ctypesgen,$(BIFROST_NAME),$(INC_DIR)) + $(call run_typehinting,$(BIFROST_NAME),$(INC_DIR)) build: bifrost/*.py Makefile $(BIFROST_PYTHON_VERSION_FILE) $(BIFROST_PYTHON_BINDINGS_FILE) $(PSRDADA_PYTHON_BINDINGS_FILE) @PYTHON@ setup.py build @PYBUILDFLAGS@ diff --git a/python/bifrost/address.py b/python/bifrost/address.py index 042386d0c..a441962f2 100644 --- a/python/bifrost/address.py +++ b/python/bifrost/address.py @@ -28,16 +28,14 @@ from bifrost.libbifrost import _bf, _check, _get, BifrostObject import ctypes -from socket import AF_UNSPEC -from typing import NewType, Optional +from socket import AddressFamily, AF_UNSPEC +from typing import Optional from bifrost import telemetry telemetry.track_module() -AFfamily_enum = NewType('AFfamily_enum', int) - class Address(BifrostObject): - def __init__(self, address: str, port: int, family: Optional[AFfamily_enum]=None): + def __init__(self, address: str, port: int, family: Optional[AddressFamily]=None): address = address.encode() assert(isinstance(port, int)) if family is None: diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index 901634f29..a42aef2b4 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -35,24 +35,13 @@ # E.g., _bf.bfRingSequenceGetName() [should be ] import ctypes -from typing import NewType import bifrost.libbifrost_generated as _bf bf = _bf # Public access to library +import bifrost.libbifrost_typehints as _th from bifrost import telemetry telemetry.track_module() -# Typing helpers - -#: Type for Bifrost return codes -BFstatus_enum = NewType('BFstatus_enum', int) - -#: Type for Bifrost memory spaces -BFspace_enum = NewType('BFspace_enum', int) - -#: Type for Bifrost data types -BFdtype_enum = NewType('BFdtype_enum', int) - # Internal helpers below class EndOfDataStop(RuntimeError): @@ -147,7 +136,7 @@ def _check(status): raise EndOfDataStop('BF_STATUS_END_OF_DATA') elif status == _bf.BF_STATUS_WOULD_BLOCK: raise IOError('BF_STATUS_WOULD_BLOCK') - return status + return _th.BFStatus_enum(status) DEREF = {ctypes.POINTER(t): t for t in [ctypes.c_bool, ctypes.c_char, diff --git a/python/typehinting.py b/python/typehinting.py new file mode 100644 index 000000000..d15c82550 --- /dev/null +++ b/python/typehinting.py @@ -0,0 +1,40 @@ +import os + +# Build a type hinting helper for libbifrost_generated.py +def build_typehinting(filename): + enums = {'status': {}, + 'space': {}, + 'dtype': {}, + 'capture': {}, + 'transmit': {}, + 'reduce': {}} + + with open(filename, 'r') as fh: + for line in fh: + if line.startswith('BF_'): + for tag in enums.keys(): + if line.startswith(f"BF_{tag.upper()}_"): + name, value = line.split('=', 1) + + name = name.strip().rstrip() + value = value.strip().rstrip() + enums[tag][name] = value + break + + outname = filename.replace('generated', 'typehints') + with open(outname, 'w') as fh: + fh.write(f""" +\"\"\" +Type hints generated from {filename} + +Do not modify this file. +\"\"\" + +import enum + +""") + for tag in enums.keys(): + fh.write(f"class BF{tag}_enum(enum.IntEnum):\n") + for key,value in enums[tag].items(): + fh.write(f" {key} = {value}\n") + fh.write("\n") From 74b4b4bda2c0503d663224c97b305172d0423e99 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Tue, 14 Feb 2023 18:16:48 -0700 Subject: [PATCH 24/56] Fixes for libbifrost.py plus an idea for Space.py. --- python/bifrost/Space.py | 38 +++++++++++------------------------- python/bifrost/libbifrost.py | 23 ++++++++-------------- 2 files changed, 19 insertions(+), 42 deletions(-) diff --git a/python/bifrost/Space.py b/python/bifrost/Space.py index b4cf4516a..a98c882af 100644 --- a/python/bifrost/Space.py +++ b/python/bifrost/Space.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2020, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,38 +25,22 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf +from bifrost.libbifrost import _bf, _th, _string2space, _space2string + +from typing import Union from bifrost import telemetry telemetry.track_module() -SPACEMAP_TO_STR = {_bf.BF_SPACE_AUTO: 'auto', - _bf.BF_SPACE_SYSTEM: 'system', - _bf.BF_SPACE_CUDA: 'cuda', - _bf.BF_SPACE_CUDA_HOST: 'cuda_host', - _bf.BF_SPACE_CUDA_MANAGED: 'cuda_managed'} - -SPACEMAP_FROM_STR = {'auto': _bf.BF_SPACE_AUTO, - 'system': _bf.BF_SPACE_SYSTEM, - 'cuda': _bf.BF_SPACE_CUDA, - 'cuda_host': _bf.BF_SPACE_CUDA_HOST, - 'cuda_managed': _bf.BF_SPACE_CUDA_MANAGED} - class Space(object): - def __init__(self, s): + def __init__(self, s: Union[str,_th.BFspace_enum,_bf.BFspace,int]): if isinstance(s, str): - if s not in set(['auto', 'system', - 'cuda', 'cuda_host', 'cuda_managed']): - raise ValueError(f"Invalid space: '{s}'") - self._space = s - elif isinstance(s, _bf.BFspace) or isinstance(s, int): - if s not in SPACEMAP_TO_STR: - raise KeyError("Invalid space: " + s + - ". Valid spaces: " + str(SPACEMAP_TO_STR.keys())) - self._space = SPACEMAP_TO_STR[s] + self._space = _string2space(s) + elif isinstance(s, (_th.BFspace_enum, _bf.BFspace, int)): + self._space = _th.BFspace_enum(s) else: raise ValueError(f"'{s}' is not a space") - def as_BFspace(self): - return SPACEMAP_FROM_STR[self._space] + def as_BFspace(self) -> _th.BFspace_enum: + return self._space def __str__(self): - return self._space + return _space2string(self._space) diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index a42aef2b4..8d399521b 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -136,7 +136,7 @@ def _check(status): raise EndOfDataStop('BF_STATUS_END_OF_DATA') elif status == _bf.BF_STATUS_WOULD_BLOCK: raise IOError('BF_STATUS_WOULD_BLOCK') - return _th.BFStatus_enum(status) + return _th.BFstatus_enum(status) DEREF = {ctypes.POINTER(t): t for t in [ctypes.c_bool, ctypes.c_char, @@ -173,21 +173,14 @@ def _get(func, *args): _check(func(*args)) return ret.value -STRING2SPACE = {'auto': _bf.BF_SPACE_AUTO, - 'system': _bf.BF_SPACE_SYSTEM, - 'cuda': _bf.BF_SPACE_CUDA, - 'cuda_host': _bf.BF_SPACE_CUDA_HOST, - 'cuda_managed': _bf.BF_SPACE_CUDA_MANAGED} def _string2space(s): - if s not in STRING2SPACE: + try: + space = getattr(_th.BFspace_enum, f"BF_SPACE_{s.upper()}") + except AttributeError: raise KeyError("Invalid space '" + str(s) + - "'.\nValid spaces: " + str(list(STRING2SPACE.keys()))) - return STRING2SPACE[s] + "'.\nValid spaces: " + str(list(_th.BFspace_enum))) + return space -SPACE2STRING = {_bf.BF_SPACE_AUTO: 'auto', - _bf.BF_SPACE_SYSTEM: 'system', - _bf.BF_SPACE_CUDA: 'cuda', - _bf.BF_SPACE_CUDA_HOST: 'cuda_host', - _bf.BF_SPACE_CUDA_MANAGED: 'cuda_managed'} def _space2string(i): - return SPACE2STRING[i] + name = _th.BFspace_enum(i).name + return name.replace('BF_SPACE_', '').lower() From 741113c83fc63b05574f251837ccea31b5a1afb4 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 10:50:55 -0700 Subject: [PATCH 25/56] Add in short names for BF_SPACE_* enums. --- python/typehinting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/typehinting.py b/python/typehinting.py index d15c82550..7bd333e5c 100644 --- a/python/typehinting.py +++ b/python/typehinting.py @@ -19,6 +19,10 @@ def build_typehinting(filename): name = name.strip().rstrip() value = value.strip().rstrip() enums[tag][name] = value + + if tag == 'space': + name = name.replace('BF_SPACE_', '') + enums[tag][name.lower()] = value break outname = filename.replace('generated', 'typehints') From 35865f946a1c4aceb45273799446896d55748aa8 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 10:53:05 -0700 Subject: [PATCH 26/56] Fix space handling in a way that works for other part of Bifrost. --- python/bifrost/Space.py | 12 ++++++------ python/bifrost/libbifrost.py | 23 +++++++++++++---------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/python/bifrost/Space.py b/python/bifrost/Space.py index a98c882af..c5ef4b0d5 100644 --- a/python/bifrost/Space.py +++ b/python/bifrost/Space.py @@ -25,7 +25,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _th, _string2space, _space2string +from bifrost.libbifrost import _bf, _th from typing import Union @@ -33,14 +33,14 @@ telemetry.track_module() class Space(object): - def __init__(self, s: Union[str,_th.BFspace_enum,_bf.BFspace,int]): + def __init__(self, s: Union[str,_th.BFspace_enum,_bf.BFspace]): if isinstance(s, str): - self._space = _string2space(s) + self._space = getattr(_th.BFspace_enum, s) elif isinstance(s, (_th.BFspace_enum, _bf.BFspace, int)): self._space = _th.BFspace_enum(s) else: raise ValueError(f"'{s}' is not a space") - def as_BFspace(self) -> _th.BFspace_enum: - return self._space + def as_BFspace(self) -> _bf.BFspace: + return _bf.BFspace(self._space.value) def __str__(self): - return _space2string(self._space) + return self._space.name diff --git a/python/bifrost/libbifrost.py b/python/bifrost/libbifrost.py index 8d399521b..5f1717fc7 100644 --- a/python/bifrost/libbifrost.py +++ b/python/bifrost/libbifrost.py @@ -36,8 +36,11 @@ import ctypes import bifrost.libbifrost_generated as _bf -bf = _bf # Public access to library import bifrost.libbifrost_typehints as _th +bf = _bf # Public access to library +th = _th # Public access to type hints + +from typing import Any, Callable from bifrost import telemetry telemetry.track_module() @@ -55,7 +58,7 @@ class EndOfDataStop(RuntimeError): class BifrostObject(object): """Base class for simple objects with create/destroy functions""" - def __init__(self, constructor, destructor, *args): + def __init__(self, constructor: Callable, destructor: Callable, *args: Any): self._obj_basename = constructor.__name__.replace('Create','') self.obj = destructor.argtypes[0]() _check(constructor(ctypes.byref(self.obj), *args)) @@ -70,14 +73,14 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self._destroy() - def set_stream(self, stream): + def set_stream(self, stream: int): set_fnc = getattr(_bf, self._obj_basename+"SetStream", None) if set_fnc is None: raise AttributeError("set_stream() is not supported by %s objects" % self._obj_basename) _check( set_fnc(self.obj, ctypes.pointer(stream)) ) - def get_stream(self): + def get_stream(self) -> int: get_fnc = getattr(_bf, self._obj_basename+"GetStream", None) if get_fnc is None: raise AttributeError("get_stream() is not supported by %s objects" % self._obj_basename) @@ -119,7 +122,7 @@ def _array(size_or_vals, dtype=None): raise TypeError("Cannot deduce C type from ", type(vals[0])) return (dtype * len(vals))(*vals) -def _check(status): +def _check(status: _bf.BFstatus) -> _th.BFstatus_enum: if __debug__: if status != _bf.BF_STATUS_SUCCESS: if status is None: @@ -165,7 +168,7 @@ def _check(status): ctypes.c_void_p, ctypes.c_wchar, ctypes.c_wchar_p]} -def _get(func, *args): +def _get(func: Callable, *args: Any) -> Any: retarg = -1 dtype = DEREF[func.argtypes[retarg]] ret = dtype() @@ -173,14 +176,14 @@ def _get(func, *args): _check(func(*args)) return ret.value -def _string2space(s): +def _string2space(s: str) -> _bf.BFspace: try: - space = getattr(_th.BFspace_enum, f"BF_SPACE_{s.upper()}") + space = getattr(_th.BFspace_enum, s) except AttributeError: raise KeyError("Invalid space '" + str(s) + "'.\nValid spaces: " + str(list(_th.BFspace_enum))) - return space + return _bf.BFspace(space.value) -def _space2string(i): +def _space2string(i: _bf.BFspace) -> str: name = _th.BFspace_enum(i).name return name.replace('BF_SPACE_', '').lower() From d411cd0e5cd3abddc4c123b4e7c04a7a2116785b Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 12:07:11 -0700 Subject: [PATCH 27/56] Do something similar for the BF_REDUCE_* enums. --- python/typehinting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/typehinting.py b/python/typehinting.py index 7bd333e5c..318399ad6 100644 --- a/python/typehinting.py +++ b/python/typehinting.py @@ -23,6 +23,10 @@ def build_typehinting(filename): if tag == 'space': name = name.replace('BF_SPACE_', '') enums[tag][name.lower()] = value + elif tag == 'reduce': + name = name.replace('BF_SPACE_', '') + name = name.replace('POWER_', 'pwr') + enums[tag][name.lower()] = value break outname = filename.replace('generated', 'typehints') From 6d5ca02cd7cb1f23fd1a866c3cac0e67b3453df2 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 14:30:42 -0700 Subject: [PATCH 28/56] First round of type hinting. --- python/bifrost/core.py | 10 +-- python/bifrost/device.py | 13 ++-- python/bifrost/dtype.py | 18 +++-- python/bifrost/fdmt.py | 22 ++++-- python/bifrost/fft.py | 16 ++-- python/bifrost/fir.py | 19 +++-- python/bifrost/guppi_raw.py | 4 +- python/bifrost/header_standard.py | 6 +- python/bifrost/linalg.py | 5 +- python/bifrost/map.py | 28 +++---- python/bifrost/memory.py | 22 +++--- python/bifrost/pipeline.py | 77 ++++++++++--------- python/bifrost/portaudio.py | 38 +++++----- python/bifrost/proclog.py | 10 ++- python/bifrost/quantize.py | 3 +- python/bifrost/reduce.py | 24 ++---- python/bifrost/ring.py | 105 ++++++++++++++------------ python/bifrost/ring2.py | 120 +++++++++++++++--------------- python/bifrost/romein.py | 9 ++- python/bifrost/sigproc.py | 28 +++---- python/bifrost/sigproc2.py | 32 ++++---- python/bifrost/temp_storage.py | 11 +-- python/bifrost/transpose.py | 6 +- python/bifrost/units.py | 6 +- python/bifrost/unpack.py | 3 +- 25 files changed, 342 insertions(+), 293 deletions(-) diff --git a/python/bifrost/core.py b/python/bifrost/core.py index 41257b232..40e233a6e 100644 --- a/python/bifrost/core.py +++ b/python/bifrost/core.py @@ -26,14 +26,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf +from bifrost.libbifrost import _bf, _th from bifrost import telemetry telemetry.track_module() -def status_string(status): - return _bf.bfGetStatusString(status) -def debug_enabled(): +def status_string(status: _th.BFstatus_enum) -> str: + return _bf.bfGetStatusString(status.value) +def debug_enabled() -> bool: return bool(_bf.bfGetDebugEnabled()) -def cuda_enabled(): +def cuda_enabled() -> bool: return bool(_bf.bfGetCudaEnabled()) diff --git a/python/bifrost/device.py b/python/bifrost/device.py index 3211b258f..26495b758 100644 --- a/python/bifrost/device.py +++ b/python/bifrost/device.py @@ -27,26 +27,27 @@ from ctypes import c_ulong, pointer as c_pointer from bifrost.libbifrost import _bf, _check, _get +from typing import Union from bifrost import telemetry telemetry.track_module() -def set_device(device): +def set_device(device: Union[int,str]) -> None: if isinstance(device, int): _check(_bf.bfDeviceSet(device)) else: _check(_bf.bfDeviceSetById(device.encode())) -def get_device(): +def get_device() -> int: return _get(_bf.bfDeviceGet) -def set_stream(stream): +def set_stream(stream: int) -> bool: """Set the CUDA stream to the provided stream handle""" stream = c_ulong(stream) _check(_bf.bfStreamSet(c_pointer(stream))) return True -def get_stream(): +def get_stream() -> int: """Get the current CUDA stream and return its address""" stream = c_ulong(0) _check(_bf.bfStreamGet(c_pointer(stream))) @@ -82,10 +83,10 @@ def __exit__(self, type, value, tb): set_stream(self._orig_stream) del self._orig_stream -def stream_synchronize(): +def stream_synchronize() -> None: _check(_bf.bfStreamSynchronize()) -def set_devices_no_spin_cpu(): +def set_devices_no_spin_cpu() -> None: """Sets a flag on all GPU devices that tells them not to spin the CPU when synchronizing. This is useful for reducing CPU load in GPU pipelines. diff --git a/python/bifrost/dtype.py b/python/bifrost/dtype.py index 66300c4b2..0d43a33d1 100644 --- a/python/bifrost/dtype.py +++ b/python/bifrost/dtype.py @@ -41,14 +41,15 @@ """ -from bifrost.libbifrost import _bf +from bifrost.libbifrost import _bf, _th from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED import numpy as np +from typing import Tuple from bifrost import telemetry telemetry.track_module() -def split_name_nbit(dtype_str): +def split_name_nbit(dtype_str: str) -> Tuple[str,int]: """Splits a dtype string into (name, nbit)""" for i, char in enumerate(dtype_str): if char.isdigit(): @@ -65,11 +66,11 @@ def split_name_nbit(dtype_str): def to_complex64(q): real_type = q.dtype['re'] return q.view(real_type).astype(np.float32).view(np.complex64) -def from_complex64(f, dtype): +def from_complex64(f, dtype: np.dtype): real_type = dtype['re'] return f.view(np.float32).astype(real_type).view(dtype) -def numpy2bifrost(dtype): +def numpy2bifrost(dtype: np.dtype) -> _th.BFdtype_enum: if dtype == np.int8: return _bf.BF_DTYPE_I8 elif dtype == np.int16: return _bf.BF_DTYPE_I16 elif dtype == np.int32: return _bf.BF_DTYPE_I32 @@ -91,7 +92,7 @@ def numpy2bifrost(dtype): and BF_FLOAT128_ENABLED: return _bf.BF_DTYPE_CF128 else: raise ValueError(f"Unsupported dtype: {dtype}") -def name_nbit2numpy(name, nbit): +def name_nbit2numpy(name: str, nbit: int) -> np.dtype: if name == 'i': if nbit == 8: return np.int8 elif nbit == 16: return np.int16 @@ -125,10 +126,11 @@ def name_nbit2numpy(name, nbit): else: raise TypeError(f"Invalid complex floating-point type size: {nbit}") else: raise TypeError(f"Invalid type name: {name}") -def string2numpy(dtype_str): + +def string2numpy(dtype_str: str) -> np.dtype: return name_nbit2numpy(*split_name_nbit(dtype_str)) -def numpy2string(dtype): +def numpy2string(dtype: np.dtype) -> str: if dtype == np.int8: return 'i8' elif dtype == np.int16: return 'i16' elif dtype == np.int32: return 'i32' @@ -146,7 +148,7 @@ def numpy2string(dtype): elif dtype == np.complex256: return 'cf128' else: raise TypeError(f"Unsupported dtype: {dtype}") -def bifrost2string(dtype): +def bifrost2string(dtype: _th.BFdtype_enum) -> str: """ Convert bifrost BF_DTYPE integer code to ndarray string """ typedict = { _bf.BF_DTYPE_I8: 'i8', diff --git a/python/bifrost/fdmt.py b/python/bifrost/fdmt.py index 20efb7be0..22050eedb 100644 --- a/python/bifrost/fdmt.py +++ b/python/bifrost/fdmt.py @@ -25,8 +25,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check, _get, BifrostObject, _string2space +from bifrost.libbifrost import _bf, _th, _check, _get, BifrostObject from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray +from bifrost.Space import Space + +from typing import Union from bifrost import telemetry telemetry.track_module() @@ -34,12 +38,13 @@ class Fdmt(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfFdmtCreate, _bf.bfFdmtDestroy) - def init(self, nchan, max_delay, f0, df, exponent=-2.0, space='cuda'): - space = _string2space(space) + def init(self, nchan: int, max_delay: int, f0: float, df: float, + exponent: float=-2.0, space: Union[str,_th.BFspace_enum,_bf.BFspace]='cuda'): + space = Space(space) psize = None _check(_bf.bfFdmtInit(self.obj, nchan, max_delay, f0, df, - exponent, space, 0, psize)) - def execute(self, idata, odata, negative_delays=False): + exponent, space.as_BFspace(), 0, psize)) + def execute(self, idata: ndarray, odata: ndarray, negative_delays: bool=False) -> ndarray: # TODO: Work out how to integrate CUDA stream psize = None _check( _bf.bfFdmtExecute( @@ -50,15 +55,16 @@ def execute(self, idata, odata, negative_delays=False): None, psize) ) return odata - def get_workspace_size(self, idata, odata): + def get_workspace_size(self, idata: ndarray, odata: ndarray) -> int: return _get(_bf.bfFdmtExecute, self.obj, asarray(idata).as_BFarray(), asarray(odata).as_BFarray(), False, None) - def execute_workspace(self, idata, odata, workspace_ptr, workspace_size, - negative_delays=False): + def execute_workspace(self, idata: ndarray, odata: ndarray, + workspace_ptr: int, workspace_size: int, + negative_delays: bool=False) -> ndarray: size = _bf.BFsize(workspace_size) _check(_bf.bfFdmtExecute( self.obj, diff --git a/python/bifrost/fft.py b/python/bifrost/fft.py index 67f2c60dc..fa1813ad2 100644 --- a/python/bifrost/fft.py +++ b/python/bifrost/fft.py @@ -25,17 +25,22 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check, _get, BifrostObject +from bifrost.libbifrost import _bf, _th, _check, _get, BifrostObject from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray import ctypes +from typing import List, Optional, Tuple, Union + from bifrost import telemetry telemetry.track_module() class Fft(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfFftCreate, _bf.bfFftDestroy) - def init(self, iarray, oarray, axes=None, apply_fftshift=False): + def init(self, iarray: ndarray, oarray: ndarray, + axes: Optional[Union[int,List[int],Tuple[int]]]=None, + apply_fftshift: bool=False): if isinstance(axes, int): axes = [axes] ndim = len(axes) @@ -49,12 +54,13 @@ def init(self, iarray, oarray, axes=None, apply_fftshift=False): ndim, axes, apply_fftshift) - def execute(self, iarray, oarray, inverse=False): + def execute(self, iarray: ndarray, oarray: ndarray, inverse: bool=False) -> ndarray: return self.execute_workspace(iarray, oarray, workspace_ptr=None, workspace_size=0, inverse=inverse) - def execute_workspace(self, iarray, oarray, workspace_ptr, workspace_size, - inverse=False): + def execute_workspace(self, iarray: ndarray, oarray: ndarray, + workspace_ptr: int, workspace_size: int, + inverse: bool=False) -> ndarray: _check(_bf.bfFftExecute( self.obj, asarray(iarray).as_BFarray(), diff --git a/python/bifrost/fir.py b/python/bifrost/fir.py index f868a1746..c728d32b0 100644 --- a/python/bifrost/fir.py +++ b/python/bifrost/fir.py @@ -26,8 +26,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check, BifrostObject, _string2space +from bifrost.libbifrost import _bf, _th, _check, BifrostObject, _string2space from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray +from bifrost.Space import Space + +from typing import Union from bifrost import telemetry telemetry.track_module() @@ -35,16 +39,17 @@ class Fir(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfFirCreate, _bf.bfFirDestroy) - def init(self, coeffs, decim=1, space='cuda'): - space = _string2space(space) + def init(self, coeffs: ndarray, decim: int=1, + space: Union[str,_th.BFspace_enum,_bf.BFspace]='cuda'): + space = Space(space) psize = None - _check( _bf.bfFirInit(self.obj, asarray(coeffs).as_BFarray(), decim, space, 0, psize) ) - def set_coeffs(self, coeffs): + _check( _bf.bfFirInit(self.obj, asarray(coeffs).as_BFarray(), decim, space.as_BFspace(), 0, psize) ) + def set_coeffs(self, coeffs: ndarray) -> None: _check( _bf.bfFirSetCoeffs(self.obj, asarray(coeffs).as_BFarray()) ) - def reset_state(self): + def reset_state(self) -> None: _check( _bf.bfFirResetState(self.obj) ) - def execute(self, idata, odata): + def execute(self, idata: ndarray, odata: ndarray) -> ndarray: # TODO: Work out how to integrate CUDA stream _check( _bf.bfFirExecute(self.obj, asarray(idata).as_BFarray(), diff --git a/python/bifrost/guppi_raw.py b/python/bifrost/guppi_raw.py index bde91560f..dbb20ad3a 100644 --- a/python/bifrost/guppi_raw.py +++ b/python/bifrost/guppi_raw.py @@ -54,10 +54,12 @@ """ +from typing import Any, Dict, IO + from bifrost import telemetry telemetry.track_module() -def read_header(f): +def read_header(f: IO[str]) -> Dict[str,Any]: RECORD_LEN = 80 DIRECTIO_ALIGN_NBYTE = 512 buf = bytearray(RECORD_LEN) diff --git a/python/bifrost/header_standard.py b/python/bifrost/header_standard.py index df438e01f..83baed275 100644 --- a/python/bifrost/header_standard.py +++ b/python/bifrost/header_standard.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,6 +45,8 @@ import numpy as np +from typing import Dict, Any + from bifrost import telemetry telemetry.track_module() @@ -61,7 +63,7 @@ 'tstart': ((float, np.float64), 0), 'tsamp': ((float, np.float64), 0)} -def enforce_header_standard(header_dict): +def enforce_header_standard(header_dict: Dict[str,Any]) -> bool: """Raise an error if the header dictionary passed does not fit the standard specified above.""" if type(header_dict) != dict: diff --git a/python/bifrost/linalg.py b/python/bifrost/linalg.py index 8b0a7116d..368e33049 100644 --- a/python/bifrost/linalg.py +++ b/python/bifrost/linalg.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,6 +27,7 @@ from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray from bifrost import telemetry telemetry.track_module() @@ -34,7 +35,7 @@ class LinAlg(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfLinAlgCreate, _bf.bfLinAlgDestroy) - def matmul(self, alpha, a, b, beta, c): + def matmul(self, alpha: float, a: ndarray, b: ndarray, beta: float, c: ndarray) -> ndarray: """Computes: c = alpha*a.b + beta*c or if b is None: diff --git a/python/bifrost/map.py b/python/bifrost/map.py index 04ab30a15..7d30164ac 100644 --- a/python/bifrost/map.py +++ b/python/bifrost/map.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,27 +25,23 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Python2 compatibility -from __future__ import absolute_import -import sys -if sys.version_info > (3,): - long = int - from bifrost.libbifrost import _bf, _check, _array from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray import numpy as np import ctypes import glob import os +from typing import Any, Dict, List, Optional from bifrost.libbifrost_generated import BF_MAP_KERNEL_DISK_CACHE from bifrost import telemetry telemetry.track_module() -def _is_literal(x): +def _is_literal(x: Any) -> bool: return isinstance(x, (int, long, float, complex)) -def _convert_to_array(arg): +def _convert_to_array(arg: Any) -> ndarray: if _is_literal(arg): arr = np.array(arg) if isinstance(arg, (int, long)) and -(1 << 31) <= arg < (1 << 31): @@ -59,9 +55,13 @@ def _convert_to_array(arg): arg = arr return asarray(arg) -def map(func_string, data, axis_names=None, shape=None, - func_name=None, extra_code=None, - block_shape=None, block_axes=None): +def map(func_string: str, data: Dict[str,Any], + axis_names: Optional[List[str]]=None, + shape: Optional[List[int]]=None, + func_name: Optional[str]=None, + extra_code: Optional[str]=None, + block_shape: Optional[List[int]]=None, + block_axes: Optional[List[int]]=None) -> ndarray: """Apply a function to a set of ndarrays. Args: @@ -146,7 +146,7 @@ def map(func_string, data, axis_names=None, shape=None, func_name, func_string, extra_code, _array(block_shape), _array(block_axes))) -def list_map_cache(): +def list_map_cache() -> None: output = "Cache enabled: %s" % ('yes' if BF_MAP_KERNEL_DISK_CACHE else 'no') if BF_MAP_KERNEL_DISK_CACHE: cache_path = os.path.join(os.path.expanduser('~'), '.bifrost', @@ -172,5 +172,5 @@ def list_map_cache(): print(output) -def clear_map_cache(): +def clear_map_cache() -> None: _check(_bf.bfMapClearCache()) diff --git a/python/bifrost/memory.py b/python/bifrost/memory.py index 2747c8919..5d89d8c9b 100644 --- a/python/bifrost/memory.py +++ b/python/bifrost/memory.py @@ -29,10 +29,12 @@ from bifrost.libbifrost import _bf, _check, _get, _string2space import ctypes +from typing import Any, List + from bifrost import telemetry telemetry.track_module() -def space_accessible(space, from_spaces): +def space_accessible(space: str, from_spaces: List[str]) -> bool: if from_spaces == 'any': # TODO: This is a little bit hacky return True from_spaces = set(from_spaces) @@ -45,27 +47,27 @@ def space_accessible(space, from_spaces): else: return False -def raw_malloc(size, space): +def raw_malloc(size: int, space: str) -> int: ptr = ctypes.c_void_p() _check(_bf.bfMalloc(ptr, size, _string2space(space))) return ptr.value -def raw_free(ptr, space='auto'): +def raw_free(ptr: int, space: str='auto') -> int: _check(_bf.bfFree(ptr, _string2space(space))) -def raw_get_space(ptr): +def raw_get_space(ptr: int) -> _bf.BFspace: return _get(_bf.bfGetSpace, ptr) -def alignment(): +def alignment() -> int: ret, _ = _bf.bfGetAlignment() return ret # **TODO: Deprecate below here! -def _get_space(arr): +def _get_space(arr: Any) -> str: try: return arr.flags['SPACE'] except KeyError: return 'system' # TODO: Dangerous to assume? # Note: These functions operate on numpy or GPU arrays -def memcpy(dst, src): +def memcpy(dst: int, src: int) -> None: assert(dst.flags['C_CONTIGUOUS']) assert(src.shape == dst.shape) dst_space = _string2space(_get_space(dst)) @@ -75,7 +77,7 @@ def memcpy(dst, src): src.ctypes.data, src_space, count)) return dst -def memcpy2D(dst, src): +def memcpy2D(dst: int, src: int) -> None: assert(len(dst.shape) == 2) assert(src.shape == dst.shape) dst_space = _string2space(_get_space(dst)) @@ -85,12 +87,12 @@ def memcpy2D(dst, src): _check(_bf.bfMemcpy2D(dst.ctypes.data, dst.strides[0], dst_space, src.ctypes.data, src.strides[0], src_space, width_bytes, height)) -def memset(dst, val=0): +def memset(dst: int, val: int=0) -> None: assert(dst.flags['C_CONTIGUOUS']) space = _string2space(_get_space(dst)) count = dst.nbytes _check(_bf.bfMemset(dst.ctypes.data, space, val, count)) -def memset2D(dst, val=0): +def memset2D(dst: int, val: int=0) -> None: assert(len(dst.shape) == 2) space = _string2space(_get_space(dst)) height, width = dst.shape diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index 57c410e51..5849597a9 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -46,6 +46,11 @@ from bifrost.ndarray import memset_array # TODO: This feels a bit hacky from bifrost.libbifrost import EndOfDataStop +from graphviz import Digraph + +from collections.abc import Iterable +from typing import Any, Callable, List, Optional, Union + from bifrost import telemetry telemetry.track_module() @@ -54,7 +59,7 @@ # module import time to make things easy. device.set_devices_no_spin_cpu() -def izip(*iterables): +def izip(*iterables: Iterable[Any]) -> List[Any]: while True: try: yield [next(it) for it in iterables] @@ -63,30 +68,30 @@ def izip(*iterables): thread_local = threading.local() thread_local.pipeline_stack = [] -def get_default_pipeline(): +def get_default_pipeline() -> "Pipeline": return thread_local.pipeline_stack[-1] thread_local.blockscope_stack = [] -def get_current_block_scope(): +def get_current_block_scope() -> "BlockScope": if len(thread_local.blockscope_stack): return thread_local.blockscope_stack[-1] else: return None -def block_scope(*args, **kwargs): +def block_scope(*args: Any, **kwargs: Any) -> "BlockScope": return BlockScope(*args, **kwargs) class BlockScope(object): instance_count = 0 def __init__(self, - name=None, - gulp_nframe=None, - buffer_nframe=None, - buffer_factor=None, - core=None, - gpu=None, - share_temp_storage=False, - fuse=False): + name: Optional[str]=None, + gulp_nframe: Optional[int]=None, + buffer_nframe: Optional[int]=None, + buffer_factor: Optional[int]=None, + core: Optional[int]=None, + gpu: Optional[int]=None, + share_temp_storage: bool=False, + fuse: bool=False): if name is None: name = f"BlockScope_{BlockScope.instance_count}" BlockScope.instance_count += 1 @@ -127,7 +132,7 @@ def __getattr__(self, name): return getattr(self._parent_scope, name) else: return None - def _get_temp_storage(self, space): + def _get_temp_storage(self, space: str) -> TempStorage: if space not in self._temp_storage_: self._temp_storage_[space] = TempStorage(space) return self._temp_storage_[space] @@ -139,25 +144,23 @@ def _get_scope_hierarchy(self): scope_hierarchy.append(parent) parent = parent._parent_scope return reversed(scope_hierarchy) - def cache_scope_hierarchy(self): + def cache_scope_hierarchy(self) -> None: self.scope_hierarchy = self._get_scope_hierarchy() self.fused_ancestor = None for ancestor in self.scope_hierarchy: if ancestor._fused: self.fused_ancestor = ancestor break - def is_fused_with(self, other): + def is_fused_with(self, other: "BlockScope") -> bool: return (self.fused_ancestor is not None and self.fused_ancestor is other.fused_ancestor) - def get_temp_storage(self, space): + def get_temp_storage(self, space: str) -> TempStorage: # TODO: Cache the first share_temp_storage scope to avoid walking each time for scope in self.scope_hierarchy: if scope.share_temp_storage: return scope._get_temp_storage(space) return self._get_temp_storage(space) - def dot_graph(self, parent_graph=None): - from graphviz import Digraph - + def dot_graph(self, parent_graph: Optional[Digraph]=None) -> Digraph: #graph_attr = {'label': self._name} graph_attr = {} if parent_graph is None: @@ -197,11 +200,11 @@ def dot_graph(self, parent_graph=None): g.subgraph(child.dot_graph()) return g -def try_join(thread, timeout=0.): +def try_join(thread: threading.Thread, timeout=0.) -> bool: thread.join(timeout) return not thread.is_alive() # Utility function for joining a collection of threads with a timeout -def join_all(threads, timeout): +def join_all(threads: List[threading.Thread], timeout: Union[int,float]): deadline = time.time() + timeout alive_threads = list(threads) while True: @@ -217,7 +220,7 @@ class PipelineInitError(Exception): class Pipeline(BlockScope): instance_count = 0 - def __init__(self, name=None, **kwargs): + def __init__(self, name: str=None, **kwargs: Any): if name is None: name = f"Pipeline_{Pipeline.instance_count}" Pipeline.instance_count += 1 @@ -228,7 +231,7 @@ def __init__(self, name=None, **kwargs): self.block_init_queue = queue.Queue() def as_default(self): return PipelineContext(self) - def synchronize_block_initializations(self): + def synchronize_block_initializations(self) -> None: # Wait for all blocks to finish initializing uninitialized_blocks = set(self.blocks) while len(uninitialized_blocks): @@ -241,7 +244,7 @@ def synchronize_block_initializations(self): f"The following block failed to initialize: {block.name}") # Tell blocks that they can begin data processing self.all_blocks_finished_initializing_event.set() - def run(self): + def run(self) -> None: # Launch blocks as threads self.threads = [threading.Thread(target=block.run, name=block.name) for block in self.blocks] @@ -254,7 +257,7 @@ def run(self): # Note: Doing it this way allows signals to be caught here while thread.is_alive(): thread.join(timeout=2**30) - def shutdown(self): + def shutdown(self) -> None: for block in self.blocks: block.shutdown() # Ensure all blocks can make progress @@ -263,7 +266,7 @@ def shutdown(self): for thread in self.threads: if thread.is_alive(): warnings.warn(f"Thread {thread.name} did not shut down on time and will be killed", RuntimeWarning) - def shutdown_on_signals(self, signals=None): + def shutdown_on_signals(self, signals: Optional[List[signal.Signals]]=None) -> None: if signals is None: signals = [signal.SIGHUP, signal.SIGINT, @@ -291,13 +294,13 @@ def __exit__(self, type, value, tb): thread_local.pipeline_stack.append(Pipeline()) thread_local.blockscope_stack.append(get_default_pipeline()) -def get_ring(block_or_ring): +def get_ring(block_or_ring: Union["Block", Ring]) -> Ring: try: return block_or_ring.orings[0] except AttributeError: return block_or_ring -def block_view(block, header_transform): +def block_view(block: "Block", header_transform: Callable) -> "Block": """View a block with modified output headers Use this function to adjust the output headers of a ring @@ -318,10 +321,10 @@ def block_view(block, header_transform): class Block(BlockScope): instance_counts = defaultdict(lambda: 0) - def __init__(self, irings, - name=None, - type_=None, - **kwargs): + def __init__(self, irings: List[Union["Block",Ring]], + name: Optional[str]=None, + type_: Optional[str]=None, + **kwargs: Any): self.type = type_ or self.__class__.__name__ self.name = name or f"{self.type}_{Block.instance_counts[self.type]}" Block.instance_counts[self.type] += 1 @@ -347,11 +350,11 @@ def __init__(self, irings, rnames[f"ring{i}"] = r.name self.in_proclog.update(rnames) self.init_trace = ''.join(traceback.format_stack()[:-1]) - def shutdown(self): + def shutdown(self) -> None: self.shutdown_event.set() - def create_ring(self, *args, **kwargs): + def create_ring(self, *args: Any, **kwargs: Any) -> Ring: return Ring(*args, owner=self, **kwargs) - def run(self): + def run(self) -> None: #affinity.set_openmp_cores(cpus) # TODO core = self.core if core is not None: @@ -370,10 +373,10 @@ def run(self): sys.stderr.write("From block instantiated here:\n") sys.stderr.write(self.init_trace) raise - def num_outputs(self): + def num_outputs(self) -> int: # TODO: This is a little hacky return len(self.orings) - def begin_writing(self, exit_stack, orings): + def begin_writing(self, exit_stack, orings: List[Ring]) -> List: return [exit_stack.enter_context(oring.begin_writing()) for oring in orings] def begin_sequences(self, exit_stack, orings, oheaders, diff --git a/python/bifrost/portaudio.py b/python/bifrost/portaudio.py index 0366e8b38..22f9b1706 100644 --- a/python/bifrost/portaudio.py +++ b/python/bifrost/portaudio.py @@ -36,6 +36,8 @@ from threading import Lock import os +from typing import Union + from bifrost import telemetry telemetry.track_module() @@ -110,7 +112,7 @@ class PaDeviceInfo(ctypes.Structure): ctypes.c_ulong] class PortAudioError(RuntimeError): - def __init__(self, msg): + def __init__(self, msg: str): super(PortAudioError, self).__init__(msg) def _check(err): @@ -118,7 +120,7 @@ def _check(err): raise PortAudioError(_lib.Pa_GetErrorText(err)) class suppress_fd(object): - def __init__(self, fd): + def __init__(self, fd: Union[str,int]): if fd.lower() == 'stdout': fd = 1 elif fd.lower() == 'stderr': fd = 2 else: assert(isinstance(fd, int)) @@ -138,13 +140,13 @@ def __exit__(self, type, value, tb): class Stream(object): def __init__(self, - mode='r', - rate=44100, - channels=2, - nbits=16, - frames_per_buffer=1024, - input_device=None, - output_device=None): + mode: str='r', + rate: int=44100, + channels: int=2, + nbits: int=16, + frames_per_buffer: int=1024, + input_device: Optional[str]=None, + output_device: Optional[str]=None): self.mode = mode self.rate = rate self.channels = channels @@ -186,7 +188,7 @@ def __init__(self, None, None)) self.start() - def close(self): + def close(self) -> None: self.stop() with self.lock: _check(_lib.Pa_CloseStream(self.stream)) @@ -194,20 +196,20 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.close() - def start(self): + def start(self) -> None: with self.lock: _check(_lib.Pa_StartStream(self.stream)) self.running = True - def stop(self): + def stop(self) -> None: with self.lock: if self.running: _check(_lib.Pa_StopStream(self.stream)) self.running = False - def read(self, nframe): + def read(self, nframe: int) -> memoryview: nbyte = nframe * self.frame_nbyte buf = ctypes.create_string_buffer("UNINITIALIZED"[:nbyte], nbyte) return self.readinto(buf) - def readinto(self, buf): + def readinto(self, buf: memoryview) -> memoryview: with self.lock: assert(len(buf) % self.frame_nbyte == 0) nframe = len(buf) // self.frame_nbyte @@ -219,21 +221,21 @@ def readinto(self, buf): # packets). _check(_lib.Pa_ReadStream(self.stream, buf_view, nframe)) return buf - def write(self, buf): + def write(self, buf: memoryview) -> memoryview: with self.lock: assert(len(buf) % self.frame_nbyte == 0) nframe = len(buf) // self.frame_nbyte buf_view = (ctypes.c_byte * len(buf)).from_buffer(buf) _check(_lib.Pa_WriteStream(self.stream, buf_view, nframe)) return buf - def time(self): + def time(self) -> int: with self.lock: return _lib.Pa_GetStreamTime(self.stream) -def open(*args, **kwargs): +def open(*args: Any, **kwargs: Any) -> Stream: return Stream(*args, **kwargs) -def get_device_count(): +def get_device_count() -> int: return _lib.Pa_GetDeviceCount() if __name__ == "__main__": diff --git a/python/bifrost/proclog.py b/python/bifrost/proclog.py index cabe4c1f1..7797186d6 100644 --- a/python/bifrost/proclog.py +++ b/python/bifrost/proclog.py @@ -30,16 +30,18 @@ import os import time +from typing import Any, Dict, Union + from bifrost import telemetry telemetry.track_module() PROCLOG_DIR = _bf.BF_PROCLOG_DIR class ProcLog(BifrostObject): - def __init__(self, name): + def __init__(self, name: str): BifrostObject.__init__( self, _bf.bfProcLogCreate, _bf.bfProcLogDestroy, name.encode()) - def update(self, contents): + def update(self, contents: Union[Dict[str,Any],str]): """Updates (replaces) the contents of the log contents: string or dict containing data to write to the log """ @@ -64,7 +66,7 @@ def _multi_convert(value): pass return value -def load_by_filename(filename): +def load_by_filename(filename: str) -> Dict[str,Any]: """ Function to read in a ProcLog file and return the contents as a dictionary. @@ -97,7 +99,7 @@ def load_by_filename(filename): # Done return contents -def load_by_pid(pid, include_rings=False): +def load_by_pid(pid: int, include_rings: bool=False) -> Dict[str,Any]: """ Function to read in and parse all ProcLog files associated with a given process ID. The contents of these files are returned as a collection of diff --git a/python/bifrost/quantize.py b/python/bifrost/quantize.py index 456c513ba..1f9f1df68 100644 --- a/python/bifrost/quantize.py +++ b/python/bifrost/quantize.py @@ -27,11 +27,12 @@ from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray from bifrost import telemetry telemetry.track_module() -def quantize(src, dst, scale=1.): +def quantize(src: ndarray, dst: ndarray, scale: float=1.) -> ndarray: src_bf = asarray(src).as_BFarray() dst_bf = asarray(dst).as_BFarray() _check(_bf.bfQuantize(src_bf, dst_bf, scale)) diff --git a/python/bifrost/reduce.py b/python/bifrost/reduce.py index 601487943..a8e9ccd9b 100644 --- a/python/bifrost/reduce.py +++ b/python/bifrost/reduce.py @@ -25,29 +25,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.libbifrost import _bf, _check +from bifrost.libbifrost import _bf, _th, _check from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray from bifrost import telemetry telemetry.track_module() -REDUCE_MAP = { - 'sum': _bf.BF_REDUCE_SUM, - 'mean': _bf.BF_REDUCE_MEAN, - 'min': _bf.BF_REDUCE_MIN, - 'max': _bf.BF_REDUCE_MAX, - 'stderr': _bf.BF_REDUCE_STDERR, - 'pwrsum': _bf.BF_REDUCE_POWER_SUM, - 'pwrmean': _bf.BF_REDUCE_POWER_MEAN, - 'pwrmin': _bf.BF_REDUCE_POWER_MIN, - 'pwrmax': _bf.BF_REDUCE_POWER_MAX, - 'pwrstderr': _bf.BF_REDUCE_POWER_STDERR, -} - -def reduce(idata, odata, op='sum'): - if op not in REDUCE_MAP: +def reduce(idata: ndarray, odata: ndarray, op: str='sum') -> ndarray: + try: + op = getattr(_th.BFreduce_enum, op) + except AttributeError: raise ValueError("Invalid reduce op: " + str(op)) - op = REDUCE_MAP[op] _check(_bf.bfReduce(asarray(idata).as_BFarray(), asarray(odata).as_BFarray(), op)) + return odata diff --git a/python/bifrost/ring.py b/python/bifrost/ring.py index 6f72c763f..5420139b3 100644 --- a/python/bifrost/ring.py +++ b/python/bifrost/ring.py @@ -36,6 +36,8 @@ import numpy as np from uuid import uuid4 +from typing import List, Optional, Tuple, Union + from bifrost import telemetry telemetry.track_module() @@ -45,7 +47,7 @@ def _slugify(name): return ''.join([c for c in name if c in valid_chars]) class Ring(BifrostObject): - def __init__(self, space='system', name=None, core=None): + def __init__(self, space: str='system', name: Optional[str]=None, core: Optional[int]=None): if name is None: name = str(uuid4()) name = _slugify(name) @@ -60,8 +62,8 @@ def __init__(self, space='system', name=None, core=None): core) ) except RuntimeError: pass - def resize(self, contiguous_span, total_span=None, nringlet=1, - buffer_factor=4): + def resize(self, contiguous_span: int, total_span: Optional[int]=None, nringlet: int=1, + buffer_factor: int=4) -> None: if total_span is None: total_span = contiguous_span * buffer_factor _check( _bf.bfRingResize(self.obj, @@ -69,13 +71,14 @@ def resize(self, contiguous_span, total_span=None, nringlet=1, total_span, nringlet) ) @property - def name(self): - return _get(_bf.bfRingGetName, self.obj) + def name(self) -> str: + n = _get(_bf.bfRingGetName, self.obj) + return n.decode() @property - def space(self): + def space(self) -> str: return _space2string(_get(_bf.bfRingGetSpace, self.obj)) @property - def core(self): + def core(self) -> int: return _get(_bf.bfRingGetAffinity, self.obj) #def begin_sequence(self, name, header="", nringlet=1): # return Sequence(ring=self, name=name, header=header, nringlet=nringlet) @@ -87,24 +90,24 @@ def core(self): # self._check( self.lib.bfRingUnlock(self.obj) ); #def lock(self): # return RingLock(self) - def begin_writing(self): + def begin_writing(self) -> "RingWriter": return RingWriter(self) def _begin_writing(self): _check( _bf.bfRingBeginWriting(self.obj) ) - def end_writing(self): + def end_writing(self) -> None: _check( _bf.bfRingEndWriting(self.obj) ) - def writing_ended(self): + def writing_ended(self) -> bool: return _get( _bf.bfRingWritingEnded, self.obj ) - def open_sequence(self, name, guarantee=True): + def open_sequence(self, name: str, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, name=name, guarantee=guarantee) - def open_sequence_at(self, time_tag, guarantee=True): + def open_sequence_at(self, time_tag: int, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='at', time_tag=time_tag, guarantee=guarantee) - def open_latest_sequence(self, guarantee=True): + def open_latest_sequence(self, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='latest', guarantee=guarantee) - def open_earliest_sequence(self, guarantee=True): + def open_earliest_sequence(self, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='earliest', guarantee=guarantee) # TODO: Alternative name? - def read(self, whence='earliest', guarantee=True): + def read(self, whence: str='earliest', guarantee: bool=True) -> "ReadSequence": with ReadSequence(self, which=whence, guarantee=guarantee) as cur_seq: while True: try: @@ -143,44 +146,46 @@ def read(self, whence='earliest', guarantee=True): # return self._get(BFsize, self.lib.bfRingLockedGetStride, self.obj) class RingWriter(object): - def __init__(self, ring): + def __init__(self, ring: Ring): self.ring = ring self.ring._begin_writing() def __enter__(self): return self def __exit__(self, type, value, tb): self.ring.end_writing() - def begin_sequence(self, name="", time_tag=-1, header="", nringlet=1): + def begin_sequence(self, name: str="", time_tag: int=-1, + header: str="", nringlet: int=1) -> "WriteSequence": return WriteSequence(ring=self.ring, name=name, time_tag=time_tag, header=header, nringlet=nringlet) class SequenceBase(object): """Python object for a ring's sequence (data unit)""" - def __init__(self, ring): + def __init__(self, ring: Ring): self._ring = ring @property def _base_obj(self): return ctypes.cast(self.obj, _bf.BFsequence) @property - def ring(self): + def ring(self) -> Ring: return self._ring @property - def name(self): - return _get(_bf.bfRingSequenceGetName, self._base_obj) + def name(self) -> str: + n = _get(_bf.bfRingSequenceGetName, self._base_obj) + return n.decode() @property - def time_tag(self): + def time_tag(self) -> int: return _get(_bf.bfRingSequenceGetTimeTag, self._base_obj) @property - def nringlet(self): + def nringlet(self) -> int: return _get(_bf.bfRingSequenceGetNRinglet, self._base_obj) @property - def header_size(self): + def header_size(self) -> int: return _get(_bf.bfRingSequenceGetHeaderSize, self._base_obj) @property def _header_ptr(self): return _get(_bf.bfRingSequenceGetHeader, self._base_obj) @property # TODO: Consider not making this a property - def header(self): + def header(self) -> np.ndarray: size = self.header_size if size == 0: # WAR for hdr_buffer_ptr.contents crashing when size == 0 @@ -193,7 +198,7 @@ def header(self): return hdr_array class WriteSequence(SequenceBase): - def __init__(self, ring, name="", time_tag=-1, header="", nringlet=1): + def __init__(self, ring: Ring, name: str="", time_tag: int=-1, header: str="", nringlet: int=1): SequenceBase.__init__(self, ring) # TODO: Allow header to be a string, buffer, or numpy array header_size = len(header) @@ -218,15 +223,16 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.end() - def end(self): + def end(self) -> None: offset_from_head = 0 _check(_bf.bfRingSequenceEnd(self.obj, offset_from_head)) - def reserve(self, size, nonblocking=False): + def reserve(self, size: int, nonblocking: bool=False) -> "WriteSpan": return WriteSpan(self.ring, size, nonblocking) class ReadSequence(SequenceBase): - def __init__(self, ring, which='specific', name="", time_tag=None, - other_obj=None, guarantee=True): + def __init__(self, ring: Ring, which: str='specific', name: str="", + time_tag: Optional[int]=None, other_obj: Optional[SequenceBase]=None, + guarantee: bool=True): SequenceBase.__init__(self, ring) self._ring = ring self.obj = _bf.BFrsequence() @@ -246,18 +252,18 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.close() - def close(self): + def close(self) -> None: _check(_bf.bfRingSequenceClose(self.obj)) #def __next__(self): # return self.next() #def next(self): # return ReadSequence(self._ring, which='next', other_obj=self.obj) - def increment(self): + def increment(self) -> None: #self._check( self.lib.bfRingSequenceNext(pointer(self.obj)) ) _check(_bf.bfRingSequenceNext(self.obj)) - def acquire(self, offset, size): + def acquire(self, offset: int, size: int) -> "ReadSpan": return ReadSpan(self, offset, size) - def read(self, span_size, stride=None, begin=0): + def read(self, span_size: int, stride: Optional[int]=None, begin: int=0) -> "ReadSpan": if stride is None: stride = span_size offset = begin @@ -270,34 +276,35 @@ def read(self, span_size, stride=None, begin=0): return class SpanBase(object): - def __init__(self, ring, writeable): + def __init__(self, ring: Ring, writeable: bool): self._ring = ring self.writeable = writeable @property def _base_obj(self): return ctypes.cast(self.obj, _bf.BFspan) @property - def ring(self): + def ring(self) -> Ring: return self._ring @property - def size(self): + def size(self) -> int: return _get(_bf.bfRingSpanGetSize, self._base_obj) @property - def stride(self): + def stride(self) -> int: return _get(_bf.bfRingSpanGetStride, self._base_obj) @property - def offset(self): + def offset(self) -> int: return _get(_bf.bfRingSpanGetOffset, self._base_obj) @property - def nringlet(self): + def nringlet(self) -> int: return _get(_bf.bfRingSpanGetNRinglet, self._base_obj) @property def _data_ptr(self): return _get(_bf.bfRingSpanGetData, self._base_obj) @property - def data(self): + def data(self) -> ndarray: return self.data_view() - def data_view(self, dtype=np.uint8, shape=-1): + def data_view(self, dtype: Union[str,np.dtype]=np.uint8, + shape: Union[int,List[int],Tuple[int]]=-1) -> ndarray: itemsize = DataType(dtype).itemsize assert( self.size % itemsize == 0 ) assert( self.stride % itemsize == 0 ) @@ -335,24 +342,24 @@ def data_view(self, dtype=np.uint8, shape=-1): class WriteSpan(SpanBase): def __init__(self, - ring, - size, - nonblocking=False): + ring: Ring, + size: int, + nonblocking: bool=False): SpanBase.__init__(self, ring, writeable=True) self.obj = _bf.BFwspan() _check(_bf.bfRingSpanReserve(self.obj, ring.obj, size, nonblocking)) self.commit_size = size - def commit(self, size): + def commit(self, size: int) -> None: self.commit_size = size def __enter__(self): return self def __exit__(self, type, value, tb): self.close() - def close(self): + def close(self) -> None: _check(_bf.bfRingSpanCommit(self.obj, self.commit_size)) class ReadSpan(SpanBase): - def __init__(self, sequence, offset, size): + def __init__(self, sequence: ReadSequence, offset: int, size: int): SpanBase.__init__(self, sequence.ring, writeable=False) self.obj = _bf.BFrspan() _check(_bf.bfRingSpanAcquire(self.obj, sequence.obj, offset, size)) @@ -360,5 +367,5 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.release() - def release(self): + def release(self) -> None: _check(_bf.bfRingSpanRelease(self.obj)) diff --git a/python/bifrost/ring2.py b/python/bifrost/ring2.py index a82f8cb15..b710da4e5 100644 --- a/python/bifrost/ring2.py +++ b/python/bifrost/ring2.py @@ -45,6 +45,9 @@ warnings.warn("Install simplejson for better performance", RuntimeWarning) import json +from collections.abc import Iterable +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + from bifrost import telemetry telemetry.track_module() @@ -54,7 +57,7 @@ def _slugify(name): return ''.join([c for c in name if c in valid_chars]) # TODO: Should probably move this elsewhere (e.g., utils) -def split_shape(shape): +def split_shape(shape: Union[List[int],Tuple[int]]) -> Tuple[List[int], List[int]]: """Splits a shape into its ringlet shape and frame shape E.g., (2,3,-1,4,5) -> (2,3), (4,5) """ @@ -66,10 +69,10 @@ def split_shape(shape): ringlet_shape.append(dim) raise ValueError("No time dimension (-1) found in shape") -def compose_unary_funcs(f, g): +def compose_unary_funcs(f: Callable, g: Callable) -> Callable: return lambda x: f(g(x)) -def ring_view(ring, header_transform): +def ring_view(ring: "Ring", header_transform: Callable) -> "Ring": new_ring = ring.view() old_header_transform = ring.header_transform if old_header_transform is not None: @@ -80,7 +83,7 @@ def ring_view(ring, header_transform): class Ring(BifrostObject): instance_count = 0 - def __init__(self, space='system', name=None, owner=None, core=None): + def __init__(self, space: str='system', name: Optional[str]=None, owner: Optional[Any]=None, core: Optional[int]=None): # If this is non-None, then the object is wrapping a base Ring instance self.base = None self.is_view = False # This gets set to True by use of .view() @@ -102,39 +105,39 @@ def __init__(self, space='system', name=None, owner=None, core=None): def __del__(self): if self.base is not None and not self.is_view: BifrostObject.__del__(self) - def view(self): + def view(self) -> "Ring": new_ring = copy(self) new_ring.base = self new_ring.is_view = True return new_ring - def resize(self, contiguous_bytes, total_bytes=None, nringlet=1): + def resize(self, contiguous_bytes: int, total_bytes: Optional[int]=None, nringlet: int=1) -> None: _check( _bf.bfRingResize(self.obj, contiguous_bytes, total_bytes, nringlet) ) @property - def name(self): + def name(self) -> str: n = _get(_bf.bfRingGetName, self.obj) return n.decode() @property - def core(self): + def core(self) -> int: return _get(_bf.bfRingGetAffinity, self.obj) - def begin_writing(self): + def begin_writing(self) -> "RingWriter": return RingWriter(self) def _begin_writing(self): _check( _bf.bfRingBeginWriting(self.obj) ) - def end_writing(self): + def end_writing(self) -> None: _check( _bf.bfRingEndWriting(self.obj) ) - def open_sequence(self, name, guarantee=True): + def open_sequence(self, name: str, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, name=name, guarantee=guarantee) - def open_sequence_at(self, time_tag, guarantee=True): + def open_sequence_at(self, time_tag: int, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='at', time_tag=time_tag, guarantee=guarantee) - def open_latest_sequence(self, guarantee=True): + def open_latest_sequence(self, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='latest', guarantee=guarantee) - def open_earliest_sequence(self, guarantee=True): + def open_earliest_sequence(self, guarantee: bool=True) -> "ReadSequence": return ReadSequence(self, which='earliest', guarantee=guarantee) # TODO: Alternative name? - def read(self, whence='earliest', guarantee=True): + def read(self, whence: str='earliest', guarantee: bool=True) -> "ReadSequence": with ReadSequence(self, which=whence, guarantee=guarantee, header_transform=self.header_transform) as cur_seq: while True: @@ -145,14 +148,14 @@ def read(self, whence='earliest', guarantee=True): return class RingWriter(object): - def __init__(self, ring): + def __init__(self, ring: Ring): self.ring = ring self.ring._begin_writing() def __enter__(self): return self def __exit__(self, type, value, tb): self.ring.end_writing() - def begin_sequence(self, header, gulp_nframe, buf_nframe): + def begin_sequence(self, header: Dict[str,Any], gulp_nframe: int, buf_nframe: int): return WriteSequence(ring=self.ring, header=header, gulp_nframe=gulp_nframe, @@ -160,7 +163,7 @@ def begin_sequence(self, header, gulp_nframe, buf_nframe): class SequenceBase(object): """Python object for a ring's sequence (data unit)""" - def __init__(self, ring): + def __init__(self, ring: Ring): self._ring = ring self._header = None self._tensor = None @@ -168,26 +171,26 @@ def __init__(self, ring): def _base_obj(self): return ctypes.cast(self.obj, _bf.BFsequence) @property - def ring(self): + def ring(self) -> Ring: return self._ring @property - def name(self): + def name(self) -> str: n = _get(_bf.bfRingSequenceGetName, self._base_obj) return n.decode() @property - def time_tag(self): + def time_tag(self) -> int: return _get(_bf.bfRingSequenceGetTimeTag, self._base_obj) @property - def nringlet(self): + def nringlet(self) -> int: return _get(_bf.bfRingSequenceGetNRinglet, self._base_obj) @property - def header_size(self): + def header_size(self) -> int: return _get(_bf.bfRingSequenceGetHeaderSize, self._base_obj) @property def _header_ptr(self): return _get(_bf.bfRingSequenceGetHeader, self._base_obj) @property - def tensor(self): # TODO: This shouldn't be public + def tensor(self) -> Dict[str,Any]: # TODO: This shouldn't be public if self._tensor is not None: return self._tensor header = self.header @@ -208,7 +211,7 @@ def tensor(self): # TODO: This shouldn't be public self._tensor['dtype_nbyte'] = nbit // 8 return self._tensor @property - def header(self): + def header(self) -> Dict[str,Any]: if self._header is not None: return self._header size = self.header_size @@ -224,7 +227,7 @@ def header(self): return self._header class WriteSequence(SequenceBase): - def __init__(self, ring, header, gulp_nframe, buf_nframe): + def __init__(self, ring: Ring, header: Dict[str,Any], gulp_nframe: int, buf_nframe: int): SequenceBase.__init__(self, ring) self._header = header # This allows passing DataType instances instead of string types @@ -254,16 +257,16 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.end() - def end(self): + def end(self) -> None: offset_from_head = 0 _check(_bf.bfRingSequenceEnd(self.obj, offset_from_head)) - def reserve(self, nframe, nonblocking=False): + def reserve(self, nframe: int, nonblocking: bool=False) -> "WriteSpan": return WriteSpan(self.ring, self, nframe, nonblocking) class ReadSequence(SequenceBase): - def __init__(self, ring, which='specific', name="", time_tag=None, - other_obj=None, guarantee=True, - header_transform=None): + def __init__(self, ring: Ring, which: str='specific', name: str="", time_tag: Optional[int]=None, + other_obj: Optional[SequenceBase]=None, guarantee: bool=True, + header_transform: Callable=None): SequenceBase.__init__(self, ring) self._ring = ring # A function for transforming the header before it's read @@ -285,17 +288,17 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.close() - def close(self): + def close(self) -> None: _check(_bf.bfRingSequenceClose(self.obj)) - def increment(self): + def increment(self) -> None: _check(_bf.bfRingSequenceNext(self.obj)) # Must invalidate cached header and tensor because this is now # a new sequence. self._header = None self._tensor = None - def acquire(self, frame_offset, nframe): + def acquire(self, frame_offset: int, nframe: int) -> "ReadSpan": return ReadSpan(self, frame_offset, nframe) - def read(self, nframe, stride=None, begin=0): + def read(self, nframe: int, stride: Optional[int]=None, begin: int=0) -> "ReadSpan": if stride is None: stride = nframe offset = begin @@ -306,7 +309,7 @@ def read(self, nframe, stride=None, begin=0): offset += stride except EndOfDataStop: return - def resize(self, gulp_nframe, buf_nframe=None, buffer_factor=None): + def resize(self, gulp_nframe: int, buf_nframe: Optional[int]=None, buffer_factor: Optional[int]=None) -> None: if buf_nframe is None: if buffer_factor is None: buffer_factor = 3 @@ -315,7 +318,7 @@ def resize(self, gulp_nframe, buf_nframe=None, buffer_factor=None): return self._ring.resize(gulp_nframe * tensor['frame_nbyte'], buf_nframe * tensor['frame_nbyte']) @property - def header(self): + def header(self) -> Dict[str,Any]: hdr = super(ReadSequence, self).header if self.header_transform is not None: hdr = self.header_transform(deepcopy(hdr)) @@ -323,11 +326,12 @@ def header(self): raise ValueError("Header transform returned None") return hdr -def accumulate(vals, op='+', init=None, reverse=False): +def accumulate(vals: Iterable[Any], op: str='+', init: Optional[Any]=None, reverse: bool=False) -> List[Any]: if op == '+': op = lambda a, b: a + b elif op == '*': op = lambda a, b: a * b elif op == 'min': op = lambda a, b: min(a, b) elif op == 'max': op = lambda a, b: max(a, b) + else: raise ValueError(f"Invalid operation '{op}'") results = [] if reverse: vals = reversed(list(vals)) @@ -341,7 +345,7 @@ def accumulate(vals, op='+', init=None, reverse=False): return results class SpanBase(object): - def __init__(self, ring, sequence, writeable): + def __init__(self, ring: Ring, sequence: SequenceBase, writeable: bool): self._ring = ring self._sequence = sequence self.writeable = writeable @@ -353,13 +357,13 @@ def _cache_info(self): self._info = _bf.BFspan_info() _check(_bf.bfRingSpanGetInfo(self._base_obj, self._info)) @property - def ring(self): + def ring(self) -> Ring: return self._ring @property - def sequence(self): + def sequence(self) -> SequenceBase: return self._sequence @property - def tensor(self): + def tensor(self) -> Dict[str,Any]: return self._sequence.tensor @property def _size_bytes(self): @@ -370,10 +374,10 @@ def _stride_bytes(self): # **TODO: Change back-end to use long instead of uint64_t return int(self._info.stride) @property - def frame_nbyte(self): + def frame_nbyte(self) -> int: return self._sequence.tensor['frame_nbyte'] @property - def frame_offset(self): + def frame_offset(self) -> int: # ***TODO: I think _info.offset could potentially not be a multiple # of frame_nbyte, because it's set to _tail when data are @@ -395,19 +399,19 @@ def _nringlet(self): def _data_ptr(self): return self._info.data @property - def nframe(self): + def nframe(self) -> int: size_bytes = self._size_bytes assert(size_bytes % self.sequence.tensor['frame_nbyte'] == 0) nframe = size_bytes // self.sequence.tensor['frame_nbyte'] return nframe @property - def shape(self): + def shape(self) -> Union[List[int],Tuple[int]]: shape = (self._sequence.tensor['ringlet_shape'] + [self.nframe] + self._sequence.tensor['frame_shape']) return shape @property - def strides(self): + def strides(self) -> List[int]: tensor = self._sequence.tensor strides = [tensor['dtype_nbyte']] for dim in reversed(tensor['frame_shape']): @@ -419,10 +423,10 @@ def strides(self): strides = list(reversed(strides)) return strides @property - def dtype(self): + def dtype(self) -> Union[str,np.dtype]: return self._sequence.tensor['dtype'] @property - def data(self): + def data(self) -> ndarray: # TODO: This function used to be super slow due to pyclibrary calls # Check whether it still is! @@ -446,10 +450,10 @@ def data(self): class WriteSpan(SpanBase): def __init__(self, - ring, - sequence, - nframe, - nonblocking=False): + ring: Ring, + sequence: WriteSequence, + nframe: int, + nonblocking: bool=False): SpanBase.__init__(self, ring, sequence, writeable=True) nbyte = nframe * self._sequence.tensor['frame_nbyte'] self.obj = _bf.BFwspan() @@ -460,19 +464,19 @@ def __init__(self, self.commit_nframe = 0 # TODO: Why do exceptions here not show up properly? #raise ValueError("SHOW ME THE ERROR") - def commit(self, nframe): + def commit(self, nframe: int) -> None: assert(nframe <= self.nframe) self.commit_nframe = nframe def __enter__(self): return self def __exit__(self, type, value, tb): self.close() - def close(self): + def close(self) -> None: commit_nbyte = self.commit_nframe * self._sequence.tensor['frame_nbyte'] _check(_bf.bfRingSpanCommit(self.obj, commit_nbyte)) class ReadSpan(SpanBase): - def __init__(self, sequence, frame_offset, nframe): + def __init__(self, sequence: ReadSequence, frame_offset: int, nframe: int): SpanBase.__init__(self, sequence.ring, sequence, writeable=False) tensor = sequence.tensor self.obj = _bf.BFrspan() @@ -485,7 +489,7 @@ def __init__(self, sequence, frame_offset, nframe): self.nframe_skipped = min(self.frame_offset - frame_offset, nframe) self.requested_frame_offset = frame_offset @property - def nframe_overwritten(self): + def nframe_overwritten(self) -> int: nbyte_overwritten = ctypes.c_ulong() _check(_bf.bfRingSpanGetSizeOverwritten(self.obj, nbyte_overwritten)) nbyte_overwritten = nbyte_overwritten.value @@ -495,5 +499,5 @@ def __enter__(self): return self def __exit__(self, type, value, tb): self.release() - def release(self): + def release(self) -> None: _check(_bf.bfRingSpanRelease(self.obj)) diff --git a/python/bifrost/romein.py b/python/bifrost/romein.py index a89c008c6..760c522f6 100644 --- a/python/bifrost/romein.py +++ b/python/bifrost/romein.py @@ -27,6 +27,7 @@ from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray from bifrost import telemetry telemetry.track_module() @@ -34,19 +35,19 @@ class Romein(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfRomeinCreate, _bf.bfRomeinDestroy) - def init(self, positions, kernels, ngrid, polmajor=True): + def init(self, positions: ndarray, kernels: ndarray, ngrid: int, polmajor: bool=True) -> None: _check( _bf.bfRomeinInit(self.obj, asarray(positions).as_BFarray(), asarray(kernels).as_BFarray(), ngrid, polmajor) ) - def set_positions(self, positions): + def set_positions(self, positions: ndarray) -> None: _check( _bf.bfRomeinSetPositions(self.obj, asarray(positions).as_BFarray()) ) - def set_kernels(self, kernels): + def set_kernels(self, kernels: ndarray) -> None: _check( _bf.bfRomeinSetKernels(self.obj, asarray(kernels).as_BFarray()) ) - def execute(self, idata, odata): + def execute(self, idata: ndarray, odata: ndarray) -> ndarray: # TODO: Work out how to integrate CUDA stream _check( _bf.bfRomeinExecute(self.obj, asarray(idata).as_BFarray(), diff --git a/python/bifrost/sigproc.py b/python/bifrost/sigproc.py index 2daf52c46..c10041bb8 100644 --- a/python/bifrost/sigproc.py +++ b/python/bifrost/sigproc.py @@ -58,6 +58,8 @@ from collections import defaultdict import os +from typing import IO, Optional + from bifrost import telemetry telemetry.track_module() @@ -205,7 +207,7 @@ def _read_header(file_object): header['header_size'] = file_object.tell() return header -def seek_to_data(file_object): +def seek_to_data(file_object: IO[bytes]) -> None: """Go the the location in the file where the data begins""" file_object.seek(0) if _header_read_one_parameter(file_object) != "HEADER_START": @@ -234,7 +236,7 @@ def seek_to_data(file_object): warnings.warn(f"Unknown header key: '{key}'", RuntimeWarning) return -def pack(data, nbit): +def pack(data: np.ndarray, nbit: int) -> np.ndarray: """downgrade data from 8bits to nbits (per value)""" data = data.flatten() if 8 % nbit != 0: @@ -246,7 +248,7 @@ def pack(data, nbit): outdata += data[index::8 // nbit] // (2**nbit)**index return outdata -def _write_data(data, nbit, file_object): +def _write_data(data: np.ndarray, nbit: int, file_object: IO[bytes]) -> np.ndarray: """Writes given data to an open file, also packing if needed""" file_object.seek(0, 2) if nbit < 8: @@ -254,7 +256,7 @@ def _write_data(data, nbit, file_object): data.tofile(file_object) # TODO: Move this elsewhere? -def unpack(data, nbit): +def unpack(data: np.ndarray, nbit: int) -> np.ndarray: """upgrade data from nbits to 8bits""" if nbit > 8: raise ValueError("unpack: nbit must be <= 8") @@ -293,7 +295,7 @@ def __init__(self): self.dtype = np.uint8 self.nbits = 8 self.header = {} - def interpret_header(self): + def interpret_header(self) -> None: """redefine variables from header dictionary""" self.nifs = self.header['nifs'] self.nchans = self.header['nchans'] @@ -320,18 +322,18 @@ def __init__(self): self.file_object = None self.mode = '' self.data = np.array([]) - def open(self, filename, mode): + def open(self, filename: str, mode: str) -> "SigprocFile": """open the filename, and read the header and data from it""" if 'b' not in mode: raise NotImplementedError("No support for non-binary files") self.mode = mode self.file_object = open(filename, mode) return self - def clear(self): + def clear(self) -> None: """Erases file contents""" self.file_object.seek(0) self.file_object.truncate() - def close(self): + def close(self) -> None: """closes file object""" self.file_object.close() def __enter__(self): @@ -345,7 +347,7 @@ def _find_nframe_from_file(self): frame_bits = self.header['nifs'] * self.header['nchans'] * self.header['nbits'] nframe = (self.file_object.tell() - curpos) * 8 // frame_bits return nframe - def get_nframe(self): + def get_nframe(self) -> int: """calculate the number of frames from the data""" if self.data.size % self.nifs != 0: raise ValueError @@ -353,11 +355,11 @@ def get_nframe(self): raise ValueError nframe = self.data.size // self.nifs // self.nchans return nframe - def read_header(self): + def read_header(self) -> None: """reads in a header from the file and sets local settings""" self.header = _read_header(self.file_object) self.interpret_header() - def read_data(self, start=None, end=None): + def read_data(self, start: Optional[int]=None, end: Optional[int]=None) -> np.ndarray: """read data from file and store it locally""" nframe = self._find_nframe_from_file() seek_to_data(self.file_object) @@ -382,12 +384,12 @@ def read_data(self, start=None, end=None): data = unpack(data, self.nbits) self.data = data return self.data - def write_to(self, filename): + def write_to(self, filename: str) -> None: """writes data and header to a different file""" with open(filename, 'wb') as file_object: _write_header(self.header, file_object) _write_data(self.data, self.nbits, file_object) - def append_data(self, input_data): + def append_data(self, input_data: np.ndarray) -> None: """append data to local data and file""" input_frames = input_data.size // self.nifs // self.nchans input_shape = (input_frames, self.nifs, self.nchans) diff --git a/python/bifrost/sigproc2.py b/python/bifrost/sigproc2.py index c21b22f23..f2b07086f 100644 --- a/python/bifrost/sigproc2.py +++ b/python/bifrost/sigproc2.py @@ -58,6 +58,8 @@ import numpy as np from collections import defaultdict +from typing import Any, Dict, IO, Optional + from bifrost import telemetry telemetry.track_module() @@ -132,14 +134,14 @@ 52: 'LWA-DP', 53: 'LWA-ADP'}) -def id2telescope(id_): +def id2telescope(id_: int) -> str: return _telescopes[id_] -def telescope2id(name): +def telescope2id(name: str) -> int: # TODO: Would be better to use a pre-made reverse lookup dict return list(_telescopes.keys())[list(_telescopes.values()).index(name)] -def id2machine(id_): +def id2machine(id_: int) -> str: return _machines[id_] -def machine2id(name): +def machine2id(name: str) -> int: # TODO: Would be better to use a pre-made reverse lookup dict return list(_machines.keys())[list(_machines.values()).index(name)] @@ -167,7 +169,7 @@ def _header_read(f): s = f.read(length) return s.decode() -def write_header(hdr, f): +def write_header(hdr: Dict[str,Any], f: IO[bytes]): _header_write_string(f, "HEADER_START") for key, val in hdr.items(): if val is None: @@ -223,7 +225,7 @@ def _read_header(f): return header # TODO: Move this elsewhere? -def unpack(data, nbit): +def unpack(data: np.ndarray, nbit: int) -> np.ndarray: if nbit > 8: raise ValueError("unpack: nbit must be <= 8") if 8 % nbit != 0: @@ -257,10 +259,10 @@ def unpack(data, nbit): # TODO: Add support for writing # Add support for data_type != filterbank class SigprocFile(object): - def __init__(self, filename=None): + def __init__(self, filename: Optional[str]=None): if filename is not None: self.open(filename) - def open(self, filename): + def open(self, filename: str) -> "SigprocFile": # Note: If nbit < 8, pack_factor = 8 // nbit and the last dimension # is divided by pack_factor, with dtype set to uint8. self.f = open(filename, 'rb') @@ -303,7 +305,7 @@ def open(self, filename): self.frame_nbit = self.frame_size * self.nbit self.frame_nbyte = self.frame_nbit // 8 return self - def close(self): + def close(self) -> None: self.f.close() def __enter__(self): return self @@ -313,14 +315,14 @@ def seek(self, offset, whence=0): if whence == 0: offset += self.header_size self.f.seek(offset, whence) - def bandwidth(self): + def bandwidth(self) -> float: return self.header['nchans'] * self.header['foff'] - def cfreq(self): + def cfreq(self) -> float: return (self.header['fch1'] + 0.5 * (self.header['nchans'] - 1) * self.header['foff']) - def duration(self): + def duration(self) -> float: return self.header['tsamp'] * self.nframe() - def nframe(self): + def nframe(self) -> int: if 'nsamples' not in self.header or self.header['nsamples'] == 0: curpos = self.f.tell() self.f.seek(0, 2) # Seek to end of file @@ -330,7 +332,7 @@ def nframe(self): self.header['nsamples'] = nframe self.f.seek(curpos, 0) # Seek back to where we were return self.header['nsamples'] - def read(self, nframe_or_start, end=None): + def read(self, nframe_or_start: int, end: Optional[int]=None) -> np.ndarray: if end is not None: start = nframe_or_start or 0 if start * self.frame_size * self.nbit % 8 != 0: @@ -368,7 +370,7 @@ def read(self, nframe_or_start, end=None): nframe = data.size // self.frame_size data = data.reshape((nframe,) + self.frame_shape) return data - def readinto(self, buf): + def readinto(self, buf: Any) -> int: """Fills buf with raw bytes straight from the file""" return self.f.readinto(buf) def __str__(self): diff --git a/python/bifrost/temp_storage.py b/python/bifrost/temp_storage.py index ee48bf863..7dc867c44 100644 --- a/python/bifrost/temp_storage.py +++ b/python/bifrost/temp_storage.py @@ -1,5 +1,5 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,14 +33,14 @@ telemetry.track_module() class TempStorage(object): - def __init__(self, space): + def __init__(self, space: str): self.space = space self.size = 0 self.ptr = None self.lock = threading.Lock() def __del__(self): self._free() - def allocate(self, size): + def allocate(self, size: int) -> "TempStorageAllocation": return TempStorageAllocation(self, size) def _allocate(self, size): if size > self.size: @@ -52,14 +52,15 @@ def _free(self): bf.memory.raw_free(self.ptr, self.space) self.ptr = None self.size = 0 + class TempStorageAllocation(object): - def __init__(self, parent, size): + def __init__(self, parent: TempStorage, size: int): self.parent = parent self.parent.lock.acquire() self.parent._allocate(size) self.size = parent.size self.ptr = parent.ptr - def release(self): + def release(self) -> None: self.parent.lock.release() def __enter__(self): return self diff --git a/python/bifrost/transpose.py b/python/bifrost/transpose.py index e84245dbb..b15722fd9 100644 --- a/python/bifrost/transpose.py +++ b/python/bifrost/transpose.py @@ -27,13 +27,16 @@ from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray import ctypes +from typing import List, Optional, Tuple, Union + from bifrost import telemetry telemetry.track_module() -def transpose(dst, src, axes=None): +def transpose(dst: ndarray, src: ndarray, axes: Optional[Union[List[int],Tuple[int]]]=None) -> ndarray: if axes is None: axes = reversed(range(len(dst.shape))) dst_bf = asarray(dst).as_BFarray() @@ -41,3 +44,4 @@ def transpose(dst, src, axes=None): array_type = ctypes.c_int * src.ndim axes_array = array_type(*axes) _check(_bf.bfTranspose(src_bf, dst_bf, axes_array)) + return dst diff --git a/python/bifrost/units.py b/python/bifrost/units.py index 5b38ba3cd..04547e3c1 100644 --- a/python/bifrost/units.py +++ b/python/bifrost/units.py @@ -27,12 +27,14 @@ import pint +from typing import Union + from bifrost import telemetry telemetry.track_module() ureg = pint.UnitRegistry() -def convert_units(value, old_units, new_units): +def convert_units(value: Union[int,float], old_units:str, new_units:str) -> Union[int,float]: old_quantity = value * ureg.parse_expression(old_units) try: new_quantity = old_quantity.to(new_units) @@ -41,7 +43,7 @@ def convert_units(value, old_units, new_units): return new_quantity.magnitude # TODO: May need something more flexible, like a Units wrapper class with __str__ -def transform_units(units, exponent): +def transform_units(units: str, exponent: Union[int,float]) -> str: old_quantity = ureg.parse_expression(units) new_quantity = old_quantity**exponent new_units_str = '{:P~}'.format(new_quantity.units) diff --git a/python/bifrost/unpack.py b/python/bifrost/unpack.py index f25dd7777..2075b980a 100644 --- a/python/bifrost/unpack.py +++ b/python/bifrost/unpack.py @@ -27,11 +27,12 @@ from bifrost.libbifrost import _bf, _check from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray from bifrost import telemetry telemetry.track_module() -def unpack(src, dst, align_msb=False): +def unpack(src: ndarray, dst: ndarray, align_msb: bool=False) -> ndarray: src_bf = asarray(src).as_BFarray() dst_bf = asarray(dst).as_BFarray() _check(_bf.bfUnpack(src_bf, dst_bf, align_msb)) From 3b15030a64356e0a3a5c434ffdcb08d3d681260b Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 14:32:35 -0700 Subject: [PATCH 29/56] No longer needed? --- python/bifrost/GPUArray.py | 122 ------------------------------------- python/bifrost/ring.py | 1 - 2 files changed, 123 deletions(-) delete mode 100644 python/bifrost/GPUArray.py diff --git a/python/bifrost/GPUArray.py b/python/bifrost/GPUArray.py deleted file mode 100644 index 504aae11e..000000000 --- a/python/bifrost/GPUArray.py +++ /dev/null @@ -1,122 +0,0 @@ - -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. -# Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of The Bifrost Authors nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -from bifrost.memory import raw_malloc, raw_free, memset, memcpy, memcpy2D -from bifrost.array import _array2bifrost # This doesn't exist! - -from bifrost import telemetry -telemetry.track_module() - -class GPUArray(object): - def __init__(self, shape, dtype, buffer=None, offset=0, strides=None): - itemsize = dtype().itemsize - shape = tuple(np.array(shape).ravel().astype(np.uint64)) - if strides is None: - # This magic came from http://stackoverflow.com/a/32874295 - strides = itemsize * np.r_[1, np.cumprod(shape[::-1][:-1], - dtype=np.int64)][::-1] - self.shape = shape - self.dtype = dtype - self.buffer = buffer - self.offset = offset - self.strides = strides - self.base = None - self.flags = {'WRITEABLE': True, - 'ALIGNED': buffer % (itemsize == 0 - if buffer is not None - else True), - 'OWNDATA': False, - 'UPDATEIFCOPY': False, - 'C_CONTIGUOUS': self.nbytes == strides[0] * shape[0], - 'F_CONTIGUOUS': False, - 'SPACE': 'cuda'} - class CTypes(object): - def __init__(self, parent): - self.parent = parent - @property - def data(self): - return self.parent.data - self.ctypes = CTypes(self) - if self.buffer is None: - self.buffer = raw_malloc(self.nbytes, space='cuda') - self.flags['OWNDATA'] = True - self.flags['ALIGNED'] = True - memset(self, 0) - else: - self.buffer += offset - def __del__(self): - if self.flags['OWNDATA']: - raw_free(self.buffer, self.flags['SPACE']) - @property - def data(self): - return self.buffer - # def reshape(self, shape): - # # TODO: How to deal with strides? - # # May be non-contiguous but the reshape still works - # # E.g., splitting dims - # return GPUArray(shape, self.dtype, - # buffer=self.buffer, - # offset=self.offset, - # strides=self.strides) - @property - def size(self): - return int(np.prod(self.shape)) - @property - def itemsize(self): - return self.dtype().itemsize - @property - def nbytes(self): - return self.size * self.itemsize - @property - def ndim(self): - return len(self.shape) - def get(self, dst=None): - hdata = dst if dst is not None else np.empty(self.shape, self.dtype) - # hdata = dst if dst is not None else np.zeros(self.shape, self.dtype) - assert(hdata.shape == self.shape) - assert(hdata.dtype == self.dtype) - if self.flags['C_CONTIGUOUS'] and hdata.flags['C_CONTIGUOUS']: - memcpy(hdata, self) - elif self.ndim == 2: - memcpy2D(hdata, self) - else: - raise RuntimeError("Copying with this data layout is unsupported") - return hdata - def set(self, hdata): - assert(hdata.shape == self.shape) - hdata = hdata.astype(self.dtype) - if self.flags['C_CONTIGUOUS'] and hdata.flags['C_CONTIGUOUS']: - memcpy(self, hdata) - elif self.ndim == 2: - memcpy2D(self, hdata) - else: - raise RuntimeError("Copying with this data layout is unsupported") - return self - def as_BFarray(self): - return _array2bifrost(self) diff --git a/python/bifrost/ring.py b/python/bifrost/ring.py index 5420139b3..53760aa44 100644 --- a/python/bifrost/ring.py +++ b/python/bifrost/ring.py @@ -27,7 +27,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from bifrost.libbifrost import _bf, _check, _get, BifrostObject, _string2space, _space2string, EndOfDataStop -#from GPUArray import GPUArray from bifrost.DataType import DataType from bifrost.ndarray import ndarray, _address_as_buffer From 6dafd2940c02e8ecf66e6ffdd91032ad854f948d Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 14:58:02 -0700 Subject: [PATCH 30/56] Too much 'Any'. --- python/bifrost/pipeline.py | 2 +- python/bifrost/ring2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index 5849597a9..b71ce3ddd 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -59,7 +59,7 @@ # module import time to make things easy. device.set_devices_no_spin_cpu() -def izip(*iterables: Iterable[Any]) -> List[Any]: +def izip(*iterables: Iterable) -> List: while True: try: yield [next(it) for it in iterables] diff --git a/python/bifrost/ring2.py b/python/bifrost/ring2.py index b710da4e5..7da7a4582 100644 --- a/python/bifrost/ring2.py +++ b/python/bifrost/ring2.py @@ -326,7 +326,7 @@ def header(self) -> Dict[str,Any]: raise ValueError("Header transform returned None") return hdr -def accumulate(vals: Iterable[Any], op: str='+', init: Optional[Any]=None, reverse: bool=False) -> List[Any]: +def accumulate(vals: Iterable, op: str='+', init: Optional=None, reverse: bool=False) -> List: if op == '+': op = lambda a, b: a + b elif op == '*': op = lambda a, b: a * b elif op == 'min': op = lambda a, b: min(a, b) From 2a147c5763ff9c9dccb75c8148af7e599c1aef16 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 15:14:24 -0700 Subject: [PATCH 31/56] Copy and paste. --- python/typehinting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/typehinting.py b/python/typehinting.py index 318399ad6..ae06c0b05 100644 --- a/python/typehinting.py +++ b/python/typehinting.py @@ -24,7 +24,7 @@ def build_typehinting(filename): name = name.replace('BF_SPACE_', '') enums[tag][name.lower()] = value elif tag == 'reduce': - name = name.replace('BF_SPACE_', '') + name = name.replace('BF_REDUCE_', '') name = name.replace('POWER_', 'pwr') enums[tag][name.lower()] = value break From fd427f48b1c98ec3d9b35438a6bf9d86e3edb92c Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 15:30:02 -0700 Subject: [PATCH 32/56] Type hinting round 2. --- python/bifrost/DataType.py | 40 ++++++++++++++--------------- python/bifrost/blocks/serialize.py | 29 ++++++++++++--------- python/bifrost/blocks/sigproc.py | 25 ++++++++++-------- python/bifrost/blocks/transpose.py | 12 ++++++--- python/bifrost/blocks/unpack.py | 16 +++++++----- python/bifrost/blocks/wav.py | 33 ++++++++++++++---------- python/bifrost/views/basic_views.py | 27 +++++++++++-------- 7 files changed, 105 insertions(+), 77 deletions(-) diff --git a/python/bifrost/DataType.py b/python/bifrost/DataType.py index 367eb004a..af63baa28 100644 --- a/python/bifrost/DataType.py +++ b/python/bifrost/DataType.py @@ -38,7 +38,7 @@ cf32: 32+32-bit complex floating point """ -from bifrost.libbifrost import _bf +from bifrost.libbifrost import _bf, _th from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED import numpy as np @@ -106,7 +106,7 @@ NUMPY_TYPEMAP['f'][128] = np.float128 NUMPY_TYPEMAP['cf'][128] = np.complex256 -def is_vector_structure(dtype): +def is_vector_structure(dtype: np.dtype) -> bool: if dtype.names is None: return False ndim = len(dtype.names) @@ -117,7 +117,7 @@ def is_vector_structure(dtype): class DataType(object): # Note: Default of None results in default Numpy type (np.float) - def __init__(self, t=None): + def __init__(self, t: Optional[Union[str,_th.BFdtype_enum,_bf.BFdtype,"DataType",np.dtype]]=None): if isinstance(t, str): for i, char in enumerate(t): if char.isdigit(): @@ -125,8 +125,8 @@ def __init__(self, t=None): self._kind = t[:i] self._nbit = int(t[i:]) self._veclen = 1 # TODO: Consider supporting this as part of string - elif isinstance(t, _bf.BFdtype): # Note: This is actually just a c_int - t = int(t) + elif isinstance(t, (_th.BFdtype_enum, _bf.BFdtype)): # Note: This is actually just a c_int + t = _th.BFdtype_enum(t).value self._nbit = t & BF_DTYPE_NBIT_BITS is_complex = bool(t & _bf.BF_DTYPE_COMPLEX_BIT) self._kind = KINDMAP[t & _bf.BF_DTYPE_TYPE_BITS] @@ -177,10 +177,10 @@ def __eq__(self, other): self._veclen == other._veclen) def __ne__(self, other): return not (self == other) - def as_BFdtype(self): + def as_BFdtype(self) -> _bf.BFdtype: base = TYPEMAP[self._kind][self._nbit] return base | ((self._veclen - 1) << _bf.BF_DTYPE_VECTOR_BIT0) - def as_numpy_dtype(self): + def as_numpy_dtype(self) -> np.dtype: base = np.dtype(NUMPY_TYPEMAP[self._kind][self._nbit]) if self._veclen == 1: return base @@ -197,21 +197,21 @@ def __str__(self): else: return f"{self._kind}{self._nbit}[{self._veclen}]" @property - def is_complex(self): + def is_complex(self) -> bool: return self._kind[0] == 'c' @property - def is_real(self): + def is_real(self) -> bool: return not self.is_complex @property - def is_signed(self): + def is_signed(self) -> bool: return 'i' in self._kind or 'f' in self._kind @property - def is_floating_point(self): + def is_floating_point(self) -> bool: return 'f' in self._kind @property - def is_integer(self): + def is_integer(self) -> bool: return 'i' in self._kind or 'u' in self._kind - def as_floating_point(self): + def as_floating_point(self) -> "DataType": """Returns the smallest floating-point type that can represent all values that self can. """ @@ -220,32 +220,32 @@ def as_floating_point(self): kind = 'cf' if self.is_complex else 'f' nbit = 32 if self._nbit <= 24 else 64 return DataType((kind, nbit, self._veclen)) - def as_integer(self, nbit=None): + def as_integer(self, nbit: int=None) -> "DataType": if nbit is None: nbit = self._nbit kind = self._kind if self.is_floating_point: kind = kind.replace('f', 'i') return DataType((kind, nbit, self._veclen)) - def as_real(self): + def as_real(self) -> "DataType": if self.is_complex: return DataType((self._kind[1:], self._nbit, self._veclen)) else: return self - def as_complex(self): + def as_complex(self) -> "DataType": if self.is_complex: return self else: return DataType(('c' + self._kind, self._nbit, self._veclen)) - def as_nbit(self, nbit): + def as_nbit(self, nbit: int) -> "DataType": return DataType((self._kind, nbit, self._veclen)) - def as_vector(self, veclen): + def as_vector(self, veclen: int) -> "DataType": return DataType((self._kind, self._nbit, veclen)) @property - def itemsize_bits(self): + def itemsize_bits(self) -> int: return self._nbit * (1 + self.is_complex) * self._veclen @property - def itemsize(self): + def itemsize(self) -> int: item_nbit = self.itemsize_bits if item_nbit < 8: raise ValueError('itemsize is undefined when nbit < 8') diff --git a/python/bifrost/blocks/serialize.py b/python/bifrost/blocks/serialize.py index 04cbc03f3..652b57856 100644 --- a/python/bifrost/blocks/serialize.py +++ b/python/bifrost/blocks/serialize.py @@ -26,6 +26,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from bifrost.pipeline import SinkBlock, SourceBlock +from bifrost.ring2 import Ring, ReadSequence, ReadSpan, WriteSpan import os import warnings try: @@ -36,6 +37,8 @@ import glob from functools import reduce +from typing import Any, Dict, List, Optional + from bifrost import telemetry telemetry.track_module() @@ -46,7 +49,7 @@ def _parse_bifrost_filename(fname): return frame0, ringlet_inds class BifrostReader(object): - def __init__(self, basename): + def __init__(self, basename: str): assert(basename.endswith('.bf')) hdr_filename = basename + '.json' with open(hdr_filename, 'r') as hdr_file: @@ -85,7 +88,7 @@ def __exit__(self, type, value, tb): else: for f in self.files: f.close() - def readinto(self, buf, frame_nbyte): + def readinto(self, buf: memoryview, frame_nbyte: int) -> int: if self.cur_file == self.nfile: return 0 nframe_read = 0 @@ -116,17 +119,18 @@ def readinto(self, buf, frame_nbyte): return nframe_read class DeserializeBlock(SourceBlock): - def __init__(self, filenames, gulp_nframe, *args, **kwargs): + def __init__(self, filenames: List[str], gulp_nframe: int, *args, **kwargs): super(DeserializeBlock, self).__init__(filenames, gulp_nframe, *args, **kwargs) - def create_reader(self, sourcename): + def create_reader(self, sourcename: str) -> BifrostReader: return BifrostReader(sourcename) - def on_sequence(self, ireader, sourcename): + def on_sequence(self, ireader: BifrostReader, sourcename: str) -> List[Dict[str,Any]]: return [ireader.header] - def on_data(self, reader, ospans): + def on_data(self, reader: BifrostReader, ospans: List[WriteSpan]) -> List[int]: ospan = ospans[0] return [reader.readinto(ospan.data, ospan.frame_nbyte)] -def deserialize(filenames, gulp_nframe, *args, **kwargs): +def deserialize(filenames: List[str], gulp_nframe: int, + *args, **kwargs) -> DeserializeBlock: """Deserializes a data stream from a set of files using a simple data format Sequence headers are read as JSON files, and sequence data are read @@ -166,7 +170,7 @@ def deserialize(filenames, gulp_nframe, *args, **kwargs): # **TODO: Write a DeserializeBlock that does the inverse of this class SerializeBlock(SinkBlock): - def __init__(self, iring, path, max_file_size=None, *args, **kwargs): + def __init__(self, iring: Ring, path: str, max_file_size: Optional[int]=None, *args, **kwargs): super(SerializeBlock, self).__init__(iring, *args, **kwargs) if path is None: path = '' @@ -197,7 +201,7 @@ def _open_new_data_files(self, frame_offset): raise NotImplementedError("Multiple ringlet axes not supported") # Open data files self.ofiles = [open(fname, 'wb') for fname in filenames] - def on_sequence(self, iseq): + def on_sequence(self, iseq: ReadSequence) -> None: hdr = iseq.header tensor = hdr['_tensor'] if hdr['name'] != '': @@ -216,9 +220,9 @@ def on_sequence(self, iseq): self.frame_axis = shape.index(-1) self.nringlet = reduce(lambda a, b: a * b, shape[:self.frame_axis], 1) self._open_new_data_files(frame_offset=0) - def on_sequence_end(self, iseq): + def on_sequence_end(self, iseq: ReadSequence) -> None: self._close_data_files() - def on_data(self, ispan): + def on_data(self, ispan: ReadSpan) -> None: if self.nringlet == 1: bytes_to_write = ispan.data.nbytes else: @@ -234,7 +238,8 @@ def on_data(self, ispan): for r in range(self.nringlet): ispan.data[r].tofile(self.ofiles[r]) -def serialize(iring, path=None, max_file_size=None, *args, **kwargs): +def serialize(iring: Ring, path: str=None, max_file_size: Optional[int]=None, + *args, **kwargs) -> SerializeBlock: """Serializes a data stream to a set of files using a simple data format Sequence headers are written as JSON files, and sequence data are written diff --git a/python/bifrost/blocks/sigproc.py b/python/bifrost/blocks/sigproc.py index ad9675f74..ada18f7a4 100644 --- a/python/bifrost/blocks/sigproc.py +++ b/python/bifrost/blocks/sigproc.py @@ -29,10 +29,13 @@ import bifrost.sigproc2 as sigproc from bifrost.DataType import DataType from bifrost.units import convert_units +from bifrost.ring2 import Ring, ReadSequence, ReadSpan, WriteSpan from numpy import transpose import os +from typing import Any, Dict, List, Optional + from bifrost import telemetry telemetry.track_module() @@ -46,12 +49,12 @@ def _unix2mjd(unix): return unix / 86400. + 40587 class SigprocSourceBlock(SourceBlock): - def __init__(self, filenames, gulp_nframe, unpack=True, *args, **kwargs): + def __init__(self, filenames: List[str], gulp_nframe: int, unpack: bool=True, *args, **kwargs): super(SigprocSourceBlock, self).__init__(filenames, gulp_nframe, *args, **kwargs) self.unpack = unpack - def create_reader(self, sourcename): + def create_reader(self, sourcename: str) -> sigproc.SigprocFile: return sigproc.SigprocFile(sourcename) - def on_sequence(self, ireader, sourcename): + def on_sequence(self, ireader: sigproc.SigprocFile, sourcename: str) -> List[Dict[str,Any]]: ihdr = ireader.header assert(ihdr['data_type'] in [1, # filterbank 2, # (dedispersed) time series @@ -98,7 +101,7 @@ def on_sequence(self, ireader, sourcename): ohdr['time_tag'] = time_tag ohdr['name'] = sourcename return [ohdr] - def on_data(self, reader, ospans): + def on_data(self, reader: sigproc.SigprocFile, ospans: List[WriteSpan]) -> List[int]: ospan = ospans[0] #print("SigprocReadBlock::on_data", ospan.data.dtype) if self.unpack: @@ -127,7 +130,8 @@ def on_data(self, reader, ospans): nframe = nbyte // ospan.frame_nbyte return [nframe] -def read_sigproc(filenames, gulp_nframe, unpack=True, *args, **kwargs): +def read_sigproc(filenames: List[str], gulp_nframe: int, unpack: bool=True, + *args, **kwargs) -> SigprocSourceBlock: """Read SIGPROC data files. Capable of reading filterbank, time series, and dedispersed subband data. @@ -156,12 +160,12 @@ def _copy_item_if_exists(dst, src, key, newkey=None): dst[newkey] = src[key] class SigprocSinkBlock(SinkBlock): - def __init__(self, iring, path=None, *args, **kwargs): + def __init__(self, iring: Ring, path: Optional[str]=None, *args, **kwargs): super(SigprocSinkBlock, self).__init__(iring, *args, **kwargs) if path is None: path = '' self.path = path - def on_sequence(self, iseq): + def on_sequence(self, iseq: ReadSequence) -> None: ihdr = iseq.header itensor = ihdr['_tensor'] @@ -316,14 +320,14 @@ def on_sequence(self, iseq): " [time, pol]\n [dispersion, time, pol]\n" + " [pol, freq, phase]") - def on_sequence_end(self, iseq): + def on_sequence_end(self, iseq: ReadSequence) -> None: if hasattr(self, 'ofile'): self.ofile.close() elif hasattr(self, 'ofiles'): for ofile in self.ofiles: ofile.close() - def on_data(self, ispan): + def on_data(self, ispan: ReadSpan) -> None: idata = ispan.data if self.data_format == 'filterbank': if len(idata.shape) == 3: @@ -353,7 +357,8 @@ def on_data(self, ispan): else: raise ValueError("Internal error: Unknown data format!") -def write_sigproc(iring, path=None, *args, **kwargs): +def write_sigproc(iring: Ring, path: Optional[str]=None, + *args, **kwargs) -> SigprocSinkBlock: """Write data as Sigproc files. Args: diff --git a/python/bifrost/blocks/transpose.py b/python/bifrost/blocks/transpose.py index a9df042d4..70cd85e93 100644 --- a/python/bifrost/blocks/transpose.py +++ b/python/bifrost/blocks/transpose.py @@ -28,19 +28,22 @@ from bifrost.transpose import transpose as bf_transpose from bifrost.memory import space_accessible from bifrost.pipeline import TransformBlock +from bifrost.ring2 import Ring, ReadSequence, ReadSpan, WriteSpan from copy import deepcopy import numpy as np +from typing import Any, Dict, List, Tuple, Union + from bifrost import telemetry telemetry.track_module() class TransposeBlock(TransformBlock): - def __init__(self, iring, axes, *args, **kwargs): + def __init__(self, iring: Ring, axes: Union[List[int],Tuple[int]], *args: Any, **kwargs: Any): super(TransposeBlock, self).__init__(iring, *args, **kwargs) self.specified_axes = axes self.space = self.orings[0].space - def on_sequence(self, iseq): + def on_sequence(self, iseq: ReadSequence) -> Dict[str,Any]: ihdr = iseq.header itensor = ihdr['_tensor'] # TODO: Is this a good idea? @@ -69,14 +72,15 @@ def on_sequence(self, iseq): otensor[item] = [itensor[item][axis] for axis in self.axes] return ohdr - def on_data(self, ispan, ospan): + def on_data(self, ispan: ReadSpan, ospan: WriteSpan) -> None: # TODO: bf.memory.transpose should support system space too if space_accessible(self.space, ['cuda']): bf_transpose(ospan.data, ispan.data, self.axes) else: ospan.data[...] = np.transpose(ispan.data, self.axes) -def transpose(iring, axes, *args, **kwargs): +def transpose(iring: Ring, axes: Union[List[int],Tuple[int]], + *args: Any, **kwargs: Any) -> TransposeBlock: """Transpose (permute) axes of the data. Args: diff --git a/python/bifrost/blocks/unpack.py b/python/bifrost/blocks/unpack.py index 4119736ad..d3bf858a6 100644 --- a/python/bifrost/blocks/unpack.py +++ b/python/bifrost/blocks/unpack.py @@ -28,22 +28,26 @@ from bifrost.unpack import unpack as bf_unpack from bifrost.pipeline import TransformBlock from bifrost.DataType import DataType +from bifrost.Ring2 import Ring, ReadSequence, ReadSpan, WriteSpan from copy import deepcopy +import numpy as np + +from typing import Any, Dict, Tuple, Union from bifrost import telemetry telemetry.track_module() class UnpackBlock(TransformBlock): - def __init__(self, iring, dtype, align_msb=False, - *args, **kwargs): + def __init__(self, iring: Ring, dtype: Union[str,np.dtype], align_msb: bool=False, + *args: Any, **kwargs: Any): super(UnpackBlock, self).__init__(iring, *args, **kwargs) self.dtype = dtype self.align_msb = align_msb - def define_valid_input_spaces(self): + def define_valid_input_spaces(self) -> Tuple[str]: """Return set of valid spaces (or 'any') for each input""" return ('system',) - def on_sequence(self, iseq): + def on_sequence(self, iseq: ReadSequence) -> Dict[str,Any]: ihdr = iseq.header ohdr = deepcopy(ihdr) itype = DataType(ihdr['_tensor']['dtype']) @@ -56,12 +60,12 @@ def on_sequence(self, iseq): otype = self.dtype ohdr['_tensor']['dtype'] = otype return ohdr - def on_data(self, ispan, ospan): + def on_data(self, ispan: ReadSpan, ospan: WriteSpan) -> None: idata = ispan.data odata = ospan.data bf_unpack(idata, odata, self.align_msb) -def unpack(iring, dtype, *args, **kwargs): +def unpack(iring: Ring, dtype: Union[str,np.dtype], *args: Any, **kwargs: Any) -> UnpackBlock: """Unpack data to a larger data type. Args: diff --git a/python/bifrost/blocks/wav.py b/python/bifrost/blocks/wav.py index 2c045681c..53a188e0c 100644 --- a/python/bifrost/blocks/wav.py +++ b/python/bifrost/blocks/wav.py @@ -28,20 +28,23 @@ from bifrost.pipeline import SourceBlock, SinkBlock from bifrost.DataType import DataType from bifrost.units import convert_units +from bifrost.ring2 import Ring, ReadSequence, WriteSpan import struct import os +from typing import Any, Dict, IO, List, Tuple + from bifrost import telemetry telemetry.track_module() -def wav_read_chunk_desc(f): +def wav_read_chunk_desc(f: IO[bytes]) -> Tuple[str,int,str]: id_, size, fmt = struct.unpack('<4sI4s', f.read(12)) return id_.decode(), size, fmt.decode() -def wav_read_subchunk_desc(f): +def wav_read_subchunk_desc(f: IO[bytes]) -> Tuple[str,size]: id_, size = struct.unpack('<4sI', f.read(8)) return id_.decode(), size -def wav_read_subchunk_fmt(f, size): +def wav_read_subchunk_fmt(f: IO[bytes], size: int) -> Dict[str,int]: assert(size >= 16) packed = f.read(16) f.seek(size - 16, 1) @@ -50,7 +53,7 @@ def wav_read_subchunk_fmt(f, size): vals = struct.unpack(' Tuple[Dict[str,int],int]: # **TODO: Some files actually have extra subchunks _after_ the data as well # This is rather annoying :/ chunk_id, chunk_size, chunk_fmt = wav_read_chunk_desc(f) @@ -66,7 +69,7 @@ def wav_read_header(f): subchunk_id, subchunk_size = wav_read_subchunk_desc(f) data_size = subchunk_size return hdr, data_size -def wav_write_header(f, hdr, chunk_size=0, data_size=0): +def wav_write_header(f: IO[bytes], hdr:Dict[str,int], chunk_size: int=0, data_size: int=0) -> None: # Note: chunk_size = file size - 8 f.write(struct.pack('<4sI4s4sIHHIIHH4sI', 'RIFF', chunk_size, 'WAVE', @@ -76,9 +79,9 @@ def wav_write_header(f, hdr, chunk_size=0, data_size=0): 'data', data_size)) class WavSourceBlock(SourceBlock): - def create_reader(self, sourcename): + def create_reader(self, sourcename: str) -> IO[bytes]: return open(sourcename, 'rb') - def on_sequence(self, reader, sourcename): + def on_sequence(self, reader: IO[bytes], sourcename: str) -> List[Dict[str,Any]]: hdr, self.bytes_remaining = wav_read_header(reader) ohdr = { '_tensor': { @@ -94,7 +97,7 @@ def on_sequence(self, reader, sourcename): } return [ohdr] - def on_data(self, reader, ospans): + def on_data(self, reader: IO[bytes], ospans: List[WriteSpan]) -> List[int]: ospan = ospans[0] nbyte = reader.readinto(ospan.data) if nbyte % ospan.frame_nbyte: @@ -109,7 +112,8 @@ def on_data(self, reader, ospans): nframe = nbyte // ospan.frame_nbyte return [nframe] -def read_wav(sourcefiles, gulp_nframe, *args, **kwargs): +def read_wav(sourcefiles: List[str], gulp_nframe: int + *args: Any, **kwargs: Any) -> WavSourceBlock: """Read Wave files (.wav). Args: @@ -129,12 +133,12 @@ def read_wav(sourcefiles, gulp_nframe, *args, **kwargs): return WavSourceBlock(sourcefiles, gulp_nframe, *args, **kwargs) class WavSinkBlock(SinkBlock): - def __init__(self, iring, path=None, *args, **kwargs): + def __init__(self, iring: Ring, path: Optional[str]=None, *args: Any, **kwargs: Any): super(WavSinkBlock, self).__init__(iring, *args, **kwargs) if path is None: path = '' self.path = path - def on_sequence(self, iseq): + def on_sequence(self, iseq: ReadSequence) -> None: ihdr = iseq.header itensor = ihdr['_tensor'] @@ -171,14 +175,14 @@ def on_sequence(self, iseq): else: raise ValueError("Incompatible axes: " + str(axnames)) - def on_sequence_end(self, iseq): + def on_sequence_end(self, iseq: ReadSequence) -> None: if hasattr(self, 'ofile'): self.ofile.close() elif hasattr(self, 'ofiles'): for ofile in self.ofiles: ofile.close() - def on_data(self, ispan): + def on_data(self, ispan: ReadSpan) -> None: idata = ispan.data if idata.ndim == 2: idata.tofile(self.ofile) @@ -188,7 +192,8 @@ def on_data(self, ispan): else: raise ValueError("Internal error: Unknown data format!") -def write_wav(iring, path=None, *args, **kwargs): +def write_wav(iring: Ring, path: Optional[str]=None, + *args: Any, **kwargs: Any) -> WavSinkBlock: """Write data as Wave files (.wav). Args: diff --git a/python/bifrost/views/basic_views.py b/python/bifrost/views/basic_views.py index 419bdc7cf..d67e17a89 100644 --- a/python/bifrost/views/basic_views.py +++ b/python/bifrost/views/basic_views.py @@ -25,27 +25,31 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from bifrost.pipeline import block_view +from bifrost.libbifrost import _bf, _th +from bifrost.pipeline import Block, block_view from bifrost.DataType import DataType from bifrost.units import convert_units -from numpy import isclose +from numpy import dtype as np_dtype, isclose + +from typing import Callable, Optional, Union from bifrost import telemetry telemetry.track_module() -def custom(block, hdr_transform): +def custom(block: Block, hdr_transform: Callable) -> Block: """An alias to `bifrost.pipeline.block_view` """ return block_view(block, hdr_transform) -def rename_axis(block, old, new): +def rename_axis(block: Block, old: str, new: str) -> Block: def header_transform(hdr, old=old, new=new): axis = hdr['_tensor']['labels'].index(old) hdr['_tensor']['labels'][axis] = new return hdr return block_view(block, header_transform) -def reinterpret_axis(block, axis, label, scale=None, units=None): +def reinterpret_axis(block: Block, axis: int, label: str, + scale: Optional[Union[int,float]]=None, units: Optional[str]=None) -> Block: """ Manually reinterpret the scale and/or units on an axis """ def header_transform(hdr, axis=axis, label=label, scale=scale, units=units): tensor = hdr['_tensor'] @@ -60,7 +64,7 @@ def header_transform(hdr, axis=axis, label=label, scale=scale, units=units): return hdr return block_view(block, header_transform) -def reverse_scale(block, axis): +def reverse_scale(block: Block, axis: int) -> Block: """ Manually reverse the scale factor on a given axis""" def header_transform(hdr, axis=axis): tensor = hdr['_tensor'] @@ -70,7 +74,8 @@ def header_transform(hdr, axis=axis): return hdr return block_view(block, header_transform) -def add_axis(block, axis, label=None, scale=None, units=None): +def add_axis(block: Block, axis: int, label: Optional[str]=None, + scale: Optional[Union[int,float]]=None, units: Optional[str]=None) -> Block: """Add an extra dimension to the frame at position 'axis' E.g., if the shape is [-1, 3, 2], then @@ -96,7 +101,7 @@ def header_transform(hdr, axis=axis, label=label, scale=scale, units=units): return hdr return block_view(block, header_transform) -def delete_axis(block, axis): +def delete_axis(block: Block, axis: int) -> Block: """Remove a unitary dimension from the frame E.g., if the shape is [-1, 1, 3, 2], then @@ -126,7 +131,7 @@ def header_transform(hdr, axis=axis): return hdr return block_view(block, header_transform) -def astype(block, dtype): +def astype(block: Block, dtype: Union[str,_th.BFdtype_enum,_bf.BFdtype,np_dtype]) -> Block: def header_transform(hdr, new_dtype=dtype): tensor = hdr['_tensor'] old_dtype = tensor['dtype'] @@ -140,7 +145,7 @@ def header_transform(hdr, new_dtype=dtype): return hdr return block_view(block, header_transform) -def split_axis(block, axis, n, label=None): +def split_axis(block: Block, axis: int, n: int, label: Optional[str]=None) -> Block: # Set function attributes to enable capture in nested function (closure) def header_transform(hdr, axis=axis, n=n, label=label): tensor = hdr['_tensor'] @@ -171,7 +176,7 @@ def header_transform(hdr, axis=axis, n=n, label=label): return hdr return block_view(block, header_transform) -def merge_axes(block, axis1, axis2, label=None): +def merge_axes(block: Block, axis1: int, axis2: int, label: Optional[str]=None) -> Block: def header_transform(hdr, axis1=axis1, axis2=axis2, label=label): tensor = hdr['_tensor'] if isinstance(axis1, str): From a8822a914e8a2845515ea3e6463982bfcacbc77d Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 15:30:39 -0700 Subject: [PATCH 33/56] Update for type hints module. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 65ddc68fe..fd4652dda 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ doc version.py /python/bifrost/version/__init__.py *_generated.py +*_typehints.py .log*.txt test/data/ *.bin From 4c9ff5d06dbddadcbe36502623785fd07f720428 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 16:03:33 -0700 Subject: [PATCH 34/56] No long in Py3. --- python/bifrost/map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/map.py b/python/bifrost/map.py index 7d30164ac..a74da1297 100644 --- a/python/bifrost/map.py +++ b/python/bifrost/map.py @@ -39,12 +39,12 @@ telemetry.track_module() def _is_literal(x: Any) -> bool: - return isinstance(x, (int, long, float, complex)) + return isinstance(x, (int, float, complex)) def _convert_to_array(arg: Any) -> ndarray: if _is_literal(arg): arr = np.array(arg) - if isinstance(arg, (int, long)) and -(1 << 31) <= arg < (1 << 31): + if isinstance(arg, int) and -(1 << 31) <= arg < (1 << 31): arr = arr.astype(np.int32) # TODO: Any way to decide when these should be double-precision? elif isinstance(arg, float): From 009bbd7b763a6f508b325724ebc69af78eac5f44 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 20:24:18 -0700 Subject: [PATCH 35/56] Another import fix. --- python/bifrost/DataType.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/bifrost/DataType.py b/python/bifrost/DataType.py index af63baa28..07928c627 100644 --- a/python/bifrost/DataType.py +++ b/python/bifrost/DataType.py @@ -42,6 +42,8 @@ from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED import numpy as np +from typing import Optional, Union + from bifrost import telemetry telemetry.track_module() From 3aa91cc491bde32627b5f44017d015e99a98ebcc Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 20:42:07 -0700 Subject: [PATCH 36/56] Typo. --- python/bifrost/blocks/unpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bifrost/blocks/unpack.py b/python/bifrost/blocks/unpack.py index d3bf858a6..053f8a95f 100644 --- a/python/bifrost/blocks/unpack.py +++ b/python/bifrost/blocks/unpack.py @@ -28,7 +28,7 @@ from bifrost.unpack import unpack as bf_unpack from bifrost.pipeline import TransformBlock from bifrost.DataType import DataType -from bifrost.Ring2 import Ring, ReadSequence, ReadSpan, WriteSpan +from bifrost.ring2 import Ring, ReadSequence, ReadSpan, WriteSpan from copy import deepcopy import numpy as np From c41af7288f41c18b495bc69e3f4708d3fdf39f88 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Wed, 15 Feb 2023 20:53:13 -0700 Subject: [PATCH 37/56] Drop Any from args and kwargs. --- python/bifrost/blocks/transpose.py | 4 ++-- python/bifrost/blocks/unpack.py | 4 ++-- python/bifrost/blocks/wav.py | 6 +++--- python/bifrost/pipeline.py | 8 ++++---- python/bifrost/portaudio.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/bifrost/blocks/transpose.py b/python/bifrost/blocks/transpose.py index 70cd85e93..85b203290 100644 --- a/python/bifrost/blocks/transpose.py +++ b/python/bifrost/blocks/transpose.py @@ -39,7 +39,7 @@ telemetry.track_module() class TransposeBlock(TransformBlock): - def __init__(self, iring: Ring, axes: Union[List[int],Tuple[int]], *args: Any, **kwargs: Any): + def __init__(self, iring: Ring, axes: Union[List[int],Tuple[int]], *args, **kwargs): super(TransposeBlock, self).__init__(iring, *args, **kwargs) self.specified_axes = axes self.space = self.orings[0].space @@ -80,7 +80,7 @@ def on_data(self, ispan: ReadSpan, ospan: WriteSpan) -> None: ospan.data[...] = np.transpose(ispan.data, self.axes) def transpose(iring: Ring, axes: Union[List[int],Tuple[int]], - *args: Any, **kwargs: Any) -> TransposeBlock: + *args, **kwargs) -> TransposeBlock: """Transpose (permute) axes of the data. Args: diff --git a/python/bifrost/blocks/unpack.py b/python/bifrost/blocks/unpack.py index 053f8a95f..4baa6a4d0 100644 --- a/python/bifrost/blocks/unpack.py +++ b/python/bifrost/blocks/unpack.py @@ -40,7 +40,7 @@ class UnpackBlock(TransformBlock): def __init__(self, iring: Ring, dtype: Union[str,np.dtype], align_msb: bool=False, - *args: Any, **kwargs: Any): + *args, **kwargs): super(UnpackBlock, self).__init__(iring, *args, **kwargs) self.dtype = dtype self.align_msb = align_msb @@ -65,7 +65,7 @@ def on_data(self, ispan: ReadSpan, ospan: WriteSpan) -> None: odata = ospan.data bf_unpack(idata, odata, self.align_msb) -def unpack(iring: Ring, dtype: Union[str,np.dtype], *args: Any, **kwargs: Any) -> UnpackBlock: +def unpack(iring: Ring, dtype: Union[str,np.dtype], *args, **kwargs) -> UnpackBlock: """Unpack data to a larger data type. Args: diff --git a/python/bifrost/blocks/wav.py b/python/bifrost/blocks/wav.py index 53a188e0c..9b78c5832 100644 --- a/python/bifrost/blocks/wav.py +++ b/python/bifrost/blocks/wav.py @@ -113,7 +113,7 @@ def on_data(self, reader: IO[bytes], ospans: List[WriteSpan]) -> List[int]: return [nframe] def read_wav(sourcefiles: List[str], gulp_nframe: int - *args: Any, **kwargs: Any) -> WavSourceBlock: + *args, **kwargs) -> WavSourceBlock: """Read Wave files (.wav). Args: @@ -133,7 +133,7 @@ def read_wav(sourcefiles: List[str], gulp_nframe: int return WavSourceBlock(sourcefiles, gulp_nframe, *args, **kwargs) class WavSinkBlock(SinkBlock): - def __init__(self, iring: Ring, path: Optional[str]=None, *args: Any, **kwargs: Any): + def __init__(self, iring: Ring, path: Optional[str]=None, *args, **kwargs): super(WavSinkBlock, self).__init__(iring, *args, **kwargs) if path is None: path = '' @@ -193,7 +193,7 @@ def on_data(self, ispan: ReadSpan) -> None: raise ValueError("Internal error: Unknown data format!") def write_wav(iring: Ring, path: Optional[str]=None, - *args: Any, **kwargs: Any) -> WavSinkBlock: + *args, **kwargs) -> WavSinkBlock: """Write data as Wave files (.wav). Args: diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index b71ce3ddd..fba605d4a 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -78,7 +78,7 @@ def get_current_block_scope() -> "BlockScope": else: return None -def block_scope(*args: Any, **kwargs: Any) -> "BlockScope": +def block_scope(*args, **kwargs) -> "BlockScope": return BlockScope(*args, **kwargs) class BlockScope(object): @@ -220,7 +220,7 @@ class PipelineInitError(Exception): class Pipeline(BlockScope): instance_count = 0 - def __init__(self, name: str=None, **kwargs: Any): + def __init__(self, name: str=None, **kwargs): if name is None: name = f"Pipeline_{Pipeline.instance_count}" Pipeline.instance_count += 1 @@ -324,7 +324,7 @@ class Block(BlockScope): def __init__(self, irings: List[Union["Block",Ring]], name: Optional[str]=None, type_: Optional[str]=None, - **kwargs: Any): + **kwargs): self.type = type_ or self.__class__.__name__ self.name = name or f"{self.type}_{Block.instance_counts[self.type]}" Block.instance_counts[self.type] += 1 @@ -352,7 +352,7 @@ def __init__(self, irings: List[Union["Block",Ring]], self.init_trace = ''.join(traceback.format_stack()[:-1]) def shutdown(self) -> None: self.shutdown_event.set() - def create_ring(self, *args: Any, **kwargs: Any) -> Ring: + def create_ring(self, *args, **kwargs) -> Ring: return Ring(*args, owner=self, **kwargs) def run(self) -> None: #affinity.set_openmp_cores(cpus) # TODO diff --git a/python/bifrost/portaudio.py b/python/bifrost/portaudio.py index 22f9b1706..5e32a7e45 100644 --- a/python/bifrost/portaudio.py +++ b/python/bifrost/portaudio.py @@ -232,7 +232,7 @@ def time(self) -> int: with self.lock: return _lib.Pa_GetStreamTime(self.stream) -def open(*args: Any, **kwargs: Any) -> Stream: +def open(*args, **kwargs) -> Stream: return Stream(*args, **kwargs) def get_device_count() -> int: From 0dd12ce68bfa85820e48d6fb03a3be11ddd63f6f Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Thu, 16 Feb 2023 11:54:33 -0700 Subject: [PATCH 38/56] Actual fixes. --- python/bifrost/blocks/wav.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/bifrost/blocks/wav.py b/python/bifrost/blocks/wav.py index 9b78c5832..16e05cc67 100644 --- a/python/bifrost/blocks/wav.py +++ b/python/bifrost/blocks/wav.py @@ -28,12 +28,12 @@ from bifrost.pipeline import SourceBlock, SinkBlock from bifrost.DataType import DataType from bifrost.units import convert_units -from bifrost.ring2 import Ring, ReadSequence, WriteSpan +from bifrost.ring2 import Ring, ReadSequence, ReadSpan, WriteSpan import struct import os -from typing import Any, Dict, IO, List, Tuple +from typing import Any, Dict, IO, List, Optional, Tuple from bifrost import telemetry telemetry.track_module() @@ -41,7 +41,7 @@ def wav_read_chunk_desc(f: IO[bytes]) -> Tuple[str,int,str]: id_, size, fmt = struct.unpack('<4sI4s', f.read(12)) return id_.decode(), size, fmt.decode() -def wav_read_subchunk_desc(f: IO[bytes]) -> Tuple[str,size]: +def wav_read_subchunk_desc(f: IO[bytes]) -> Tuple[str,int]: id_, size = struct.unpack('<4sI', f.read(8)) return id_.decode(), size def wav_read_subchunk_fmt(f: IO[bytes], size: int) -> Dict[str,int]: @@ -112,7 +112,7 @@ def on_data(self, reader: IO[bytes], ospans: List[WriteSpan]) -> List[int]: nframe = nbyte // ospan.frame_nbyte return [nframe] -def read_wav(sourcefiles: List[str], gulp_nframe: int +def read_wav(sourcefiles: List[str], gulp_nframe: int, *args, **kwargs) -> WavSourceBlock: """Read Wave files (.wav). From ef644d34c4b0c8344ff3cf6cc4a3c588a7357979 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 22 Apr 2022 09:53:32 -0600 Subject: [PATCH 39/56] Removed commented out code in ring.py Cherry-picked from c2778a1ee1381adcb79c7ceef7ca85327d96c64f on branch apr2022-prune, PR #178 --- python/bifrost/ring.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/python/bifrost/ring.py b/python/bifrost/ring.py index 53760aa44..50bd24854 100644 --- a/python/bifrost/ring.py +++ b/python/bifrost/ring.py @@ -114,35 +114,6 @@ def read(self, whence: str='earliest', guarantee: bool=True) -> "ReadSequence": cur_seq.increment() except EndOfDataStop: return - #def _data(self): - # data_ptr = _get(self.lib.bfRingLockedGetData, self.obj) - # #data_ptr = c_void_p() - # #self._check( self.lib.bfRingLockedGetData(self.obj, pointer(data_ptr)) ) - # #data_ptr = data_ptr.value - # #data_ptr = self.lib.bfRingLockedGetData(self.obj) - # if self.space == 'cuda': - # # TODO: See if can wrap this in something like PyCUDA's GPUArray - # # Ideally actual GPUArray, but it doesn't appear to support wrapping pointers - # return data_ptr - # span = self._total_span() - # stride = self._stride() - # nringlet = self._nringlet() - # #print("******", span, stride, nringlet) - # BufferType = c_byte*(nringlet*stride) - # data_buffer_ptr = cast(data_ptr, POINTER(BufferType)) - # data_buffer = data_buffer_ptr.contents - # data_array = np.ndarray(shape=(nringlet, span), - # strides=(stride, 1) if nringlet > 1 else None, - # buffer=data_buffer, dtype=np.uint8) - # return data_array - #def _contiguous_span(self): - # return self._get(BFsize, self.lib.bfRingLockedGetContiguousSpan, self.obj) - #def _total_span(self): - # return self._get(BFsize, self.lib.bfRingLockedGetTotalSpan, self.obj) - #def _nringlet(self): - # return self._get(BFsize, self.lib.bfRingLockedGetNRinglet, self.obj) - #def _stride(self): - # return self._get(BFsize, self.lib.bfRingLockedGetStride, self.obj) class RingWriter(object): def __init__(self, ring: Ring): @@ -308,15 +279,8 @@ def data_view(self, dtype: Union[str,np.dtype]=np.uint8, assert( self.size % itemsize == 0 ) assert( self.stride % itemsize == 0 ) data_ptr = self._data_ptr - #if self.sequence.ring.space == 'cuda': - # # TODO: See if can wrap this in something like PyCUDA's GPUArray - # # Ideally actual GPUArray, but it doesn't appear to support wrapping pointers - # # Could also try writing a custom GPUArray implem for this purpose - # return data_ptr span_size = self.size nringlet = self.nringlet - #print("******", span_size, stride, nringlet) - #BufferType = c_byte*(span_size*self.stride) # TODO: We should really map the actual ring memory space and index # it with offset rather than mapping from the current pointer. _shape = (nringlet, span_size // itemsize) From 8cdc0c7739790d5f22be2071ca1ba8939f9034d3 Mon Sep 17 00:00:00 2001 From: Christopher League Date: Thu, 20 Jul 2023 19:26:24 -0600 Subject: [PATCH 40/56] Remove dtype in favor of DataType This supersedes e6bed74a50098a3adaba37ddbffa6a313369a033 in apr2022-prune branch (PR #178). There was a new usage of dtype.bifrost2string in ndarray.py, which wasn't part of the apr2022-prune branch. I think (?) I worked around it correctly using the enum. We also discovered and fixed a NameError bug in DataType. --- python/bifrost/DataType.py | 2 +- python/bifrost/dtype.py | 180 ------------------------------------- python/bifrost/ndarray.py | 5 +- 3 files changed, 3 insertions(+), 184 deletions(-) delete mode 100644 python/bifrost/dtype.py diff --git a/python/bifrost/DataType.py b/python/bifrost/DataType.py index 07928c627..72690b362 100644 --- a/python/bifrost/DataType.py +++ b/python/bifrost/DataType.py @@ -129,7 +129,7 @@ def __init__(self, t: Optional[Union[str,_th.BFdtype_enum,_bf.BFdtype,"DataType" self._veclen = 1 # TODO: Consider supporting this as part of string elif isinstance(t, (_th.BFdtype_enum, _bf.BFdtype)): # Note: This is actually just a c_int t = _th.BFdtype_enum(t).value - self._nbit = t & BF_DTYPE_NBIT_BITS + self._nbit = t & _bf.BF_DTYPE_NBIT_BITS is_complex = bool(t & _bf.BF_DTYPE_COMPLEX_BIT) self._kind = KINDMAP[t & _bf.BF_DTYPE_TYPE_BITS] if is_complex: diff --git a/python/bifrost/dtype.py b/python/bifrost/dtype.py deleted file mode 100644 index 0d43a33d1..000000000 --- a/python/bifrost/dtype.py +++ /dev/null @@ -1,180 +0,0 @@ - -# Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of The Bifrost Authors nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" - -i: signed integer -u: unsigned integer -f: floating point -ci: complex signed integer -cu: complex unsigned integer -cf: complex floating pointer - -i4: 4-bit signed integer -f16: 16-bit floating point -ci4: 4+4-bit complex signed integer -cf32: 32+32-bit complex floating point - -""" - -from bifrost.libbifrost import _bf, _th -from bifrost.libbifrost_generated import BF_FLOAT128_ENABLED -import numpy as np -from typing import Tuple - -from bifrost import telemetry -telemetry.track_module() - -def split_name_nbit(dtype_str: str) -> Tuple[str,int]: - """Splits a dtype string into (name, nbit)""" - for i, char in enumerate(dtype_str): - if char.isdigit(): - break - name = dtype_str[:i] - nbit = int(dtype_str[i:]) - return name, nbit - -# Custom dtypes to represent additional complex types -ci8 = np.dtype([('re', np.int8), ('im', np.int8)]) -ci16 = np.dtype([('re', np.int16), ('im', np.int16)]) -ci32 = np.dtype([('re', np.int32), ('im', np.int32)]) -cf16 = np.dtype([('re', np.float16), ('im', np.float16)]) -def to_complex64(q): - real_type = q.dtype['re'] - return q.view(real_type).astype(np.float32).view(np.complex64) -def from_complex64(f, dtype: np.dtype): - real_type = dtype['re'] - return f.view(np.float32).astype(real_type).view(dtype) - -def numpy2bifrost(dtype: np.dtype) -> _th.BFdtype_enum: - if dtype == np.int8: return _bf.BF_DTYPE_I8 - elif dtype == np.int16: return _bf.BF_DTYPE_I16 - elif dtype == np.int32: return _bf.BF_DTYPE_I32 - elif dtype == np.uint8: return _bf.BF_DTYPE_U8 - elif dtype == np.uint16: return _bf.BF_DTYPE_U16 - elif dtype == np.uint32: return _bf.BF_DTYPE_U32 - elif dtype == np.float16: return _bf.BF_DTYPE_F16 - elif dtype == np.float32: return _bf.BF_DTYPE_F32 - elif dtype == np.float64: return _bf.BF_DTYPE_F64 - elif dtype == np.float128 \ - and BF_FLOAT128_ENABLED: return _bf.BF_DTYPE_F128 - elif dtype == ci8: return _bf.BF_DTYPE_CI8 - elif dtype == ci16: return _bf.BF_DTYPE_CI16 - elif dtype == ci32: return _bf.BF_DTYPE_CI32 - elif dtype == cf16: return _bf.BF_DTYPE_CF16 - elif dtype == np.complex64: return _bf.BF_DTYPE_CF32 - elif dtype == np.complex128: return _bf.BF_DTYPE_CF64 - elif dtype == np.complex256 \ - and BF_FLOAT128_ENABLED: return _bf.BF_DTYPE_CF128 - else: raise ValueError(f"Unsupported dtype: {dtype}") - -def name_nbit2numpy(name: str, nbit: int) -> np.dtype: - if name == 'i': - if nbit == 8: return np.int8 - elif nbit == 16: return np.int16 - elif nbit == 32: return np.int32 - elif nbit == 64: return np.int64 - else: raise TypeError(f"Invalid signed integer type size: {nbit}") - elif name == 'u': - if nbit == 8: return np.uint8 - elif nbit == 16: return np.uint16 - elif nbit == 32: return np.uint32 - elif nbit == 64: return np.uint64 - else: raise TypeError(f"Invalid unsigned integer type size: {nbit}") - elif name == 'f': - if nbit == 16: return np.float16 - elif nbit == 32: return np.float32 - elif nbit == 64: return np.float64 - elif nbit == 128: return np.float128 - else: raise TypeError(f"Invalid floating-point type size: {nbit}") - elif name == 'ci': - if nbit == 8: return ci8 - elif nbit == 16: return ci16 - elif nbit == 32: return ci32 - # elif name in set(['ci', 'cu']): - # Note: This gives integer types in place of proper complex types - # return name_nbit2numpy(name[1:], nbit*2) - elif name == 'cf': - if nbit == 16: return cf16 - elif nbit == 32: return np.complex64 - elif nbit == 64: return np.complex128 - elif nbit == 128: return np.complex256 - else: raise TypeError(f"Invalid complex floating-point type size: {nbit}") - else: - raise TypeError(f"Invalid type name: {name}") - -def string2numpy(dtype_str: str) -> np.dtype: - return name_nbit2numpy(*split_name_nbit(dtype_str)) - -def numpy2string(dtype: np.dtype) -> str: - if dtype == np.int8: return 'i8' - elif dtype == np.int16: return 'i16' - elif dtype == np.int32: return 'i32' - elif dtype == np.int64: return 'i64' - elif dtype == np.uint8: return 'u8' - elif dtype == np.uint16: return 'u16' - elif dtype == np.uint32: return 'u32' - elif dtype == np.uint64: return 'u64' - elif dtype == np.float16: return 'f16' - elif dtype == np.float32: return 'f32' - elif dtype == np.float64: return 'f64' - elif dtype == np.float128: return 'f128' - elif dtype == np.complex64: return 'cf32' - elif dtype == np.complex128: return 'cf64' - elif dtype == np.complex256: return 'cf128' - else: raise TypeError(f"Unsupported dtype: {dtype}") - -def bifrost2string(dtype: _th.BFdtype_enum) -> str: - """ Convert bifrost BF_DTYPE integer code to ndarray string """ - typedict = { - _bf.BF_DTYPE_I8: 'i8', - _bf.BF_DTYPE_I16: 'i16', - _bf.BF_DTYPE_I32: 'i32', - _bf.BF_DTYPE_I64: 'i64', - _bf.BF_DTYPE_U8: 'u8', - _bf.BF_DTYPE_U16: 'u16', - _bf.BF_DTYPE_U32: 'u32', - _bf.BF_DTYPE_U64: 'u64', - _bf.BF_DTYPE_F16: 'f16', - _bf.BF_DTYPE_F32: 'f32', - _bf.BF_DTYPE_F64: 'f64', - _bf.BF_DTYPE_CI8: 'ci8', - _bf.BF_DTYPE_CI16: 'ci16', - _bf.BF_DTYPE_CI32: 'ci32', - _bf.BF_DTYPE_CF16: 'cf16', - _bf.BF_DTYPE_CF32: 'cf32', - _bf.BF_DTYPE_CF64: 'cf64', - } - if BF_FLOAT128_ENABLED: - typedict[_bf.BF_DTYPE_CF128] = 'cf128' - typedict[_bf.BF_DTYPE_F128] = 'f128' - - dtype_str = typedict.get(dtype) - if dtype_str is None: - raise ValueError("Could not convert dtype integer to string. Value not understood.") - else: - return dtype_str diff --git a/python/bifrost/ndarray.py b/python/bifrost/ndarray.py index d632b0897..98f9da658 100644 --- a/python/bifrost/ndarray.py +++ b/python/bifrost/ndarray.py @@ -40,9 +40,8 @@ import ctypes import numpy as np from bifrost.memory import raw_malloc, raw_free, raw_get_space, space_accessible -from bifrost.libbifrost import _bf, _check, _space2string +from bifrost.libbifrost import _bf, _th, _check, _space2string from bifrost import device -from bifrost.dtype import bifrost2string from bifrost.DataType import DataType from bifrost.Space import Space from bifrost.libbifrost_generated import struct_BFarray_ @@ -171,7 +170,7 @@ def __new__(cls, base=None, space=None, shape=None, dtype=None, shape = list(base.shape)[:ndim] strides = list(base.strides)[:ndim] space = _space2string(base.space) - dtype = bifrost2string(base.dtype) + dtype = _th.BFdtype_enum(base.dtype) return ndarray.__new__(cls, space=space, From 6faac9521dca809ced49b9820dc8710a00c65770 Mon Sep 17 00:00:00 2001 From: Christopher League Date: Thu, 20 Jul 2023 19:41:43 -0600 Subject: [PATCH 41/56] Oops, somehow this didn't make it into last commit --- python/bifrost/blocks/binary_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/blocks/binary_io.py b/python/bifrost/blocks/binary_io.py index c34766166..bbcbba515 100644 --- a/python/bifrost/blocks/binary_io.py +++ b/python/bifrost/blocks/binary_io.py @@ -32,7 +32,7 @@ """ import numpy as np import bifrost.pipeline as bfp -from bifrost.dtype import name_nbit2numpy +from bifrost.DataType import DataType from bifrost import telemetry telemetry.track_module() @@ -76,7 +76,7 @@ def create_reader(self, filename): # Do a lookup on bifrost datatype to numpy datatype dcode = self.dtype.rstrip('0123456789') nbits = int(self.dtype[len(dcode):]) - np_dtype = name_nbit2numpy(dcode, nbits) + np_dtype = DataType(dcode+str(nbits)).as_numpy_dtype() return BinaryFileRead(filename, self.gulp_size, np_dtype) From 8cff5c27be4c7d8cae70d9f9f561a42e7d435b14 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 21 Jul 2023 10:11:49 -0600 Subject: [PATCH 42/56] Switch to `Run(exit=False)` for #214. --- test/test_scripts.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 333eb9ca9..03a061932 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -62,11 +62,7 @@ def _test_script(self, script): pylint_output = StringIO() reporter = TextReporter(pylint_output) - try: - Run([script, '-E', '--extension-pkg-whitelist=numpy'], reporter=reporter, do_exit=False) - except TypeError: - # Python2 catch - Run([script, '-E', '--extension-pkg-whitelist=numpy'], reporter=reporter) + Run([script, '-E', '--extension-pkg-whitelist=numpy'], reporter=reporter, exit=False) out = pylint_output.getvalue() out_lines = out.split('\n') From 26637e25413b0eb5e44ff92c3dd7e5e74f47dfe7 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 21 Jul 2023 10:13:43 -0600 Subject: [PATCH 43/56] `.data` is 2D so try `.data[0,0]` for #214. --- test/test_block.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_block.py b/test/test_block.py index d7ab481b4..3382bea79 100644 --- a/test/test_block.py +++ b/test/test_block.py @@ -476,11 +476,11 @@ def monitor_block_sequences(array): if self.i > 1 and self.i < 11: with self.monitor_block.rings['out_1'].open_latest_sequence(guarantee=False) as curr_seq: span_gen = curr_seq.read(1) - self.all_sequence_starts.append(int(next(span_gen).data[0])) + self.all_sequence_starts.append(int(next(span_gen).data[0,0])) if self.i > 12: with self.monitor_block.rings['out_1'].open_latest_sequence(guarantee=False) as curr_seq: span_gen = curr_seq.read(1) - self.all_sequence_starts.append(int(next(span_gen).data[0])) + self.all_sequence_starts.append(int(next(span_gen).data[0,0])) self.i += 1 return array From f25ee950a436e3507d6e49f2d8b53d7147a0dd7d Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 21 Jul 2023 10:23:14 -0600 Subject: [PATCH 44/56] Another str -> bytes-related Py2 catch. --- python/bifrost/map.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/python/bifrost/map.py b/python/bifrost/map.py index a74da1297..cb11189c3 100644 --- a/python/bifrost/map.py +++ b/python/bifrost/map.py @@ -110,15 +110,11 @@ def map(func_string: str, data: Dict[str,Any], # Slice an array with a scalar index bf.map("c(i) = a(i,k)", {'c': c, 'a': a, 'k': 7}, ['i'], shape=c.shape) """ - try: - func_string = func_string.encode() - if func_name is not None: - func_name = func_name.encode() - if extra_code is not None: - extra_code = extra_code.encode() - except AttributeError: - # Python2 catch - pass + func_string = func_string.encode() + if func_name is not None: + func_name = func_name.encode() + if extra_code is not None: + extra_code = extra_code.encode() narg = len(data) ndim = len(shape) if shape is not None else 0 arg_arrays = [] From 30a1aa6228125a619347cae8b1e01687ceed166e Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 13:35:11 -0600 Subject: [PATCH 45/56] Date fix. --- python/bifrost/udp_transmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/bifrost/udp_transmit.py b/python/bifrost/udp_transmit.py index 5bb4c0060..f7bed1b71 100644 --- a/python/bifrost/udp_transmit.py +++ b/python/bifrost/udp_transmit.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017-202, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. # Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without From b5f503a7ca8cd0808330c00581ed8662703a07b0 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 13:40:18 -0600 Subject: [PATCH 46/56] Require Python 3.6+. --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index 3c0ba4c22..d7546bf85 100755 --- a/python/setup.py +++ b/python/setup.py @@ -63,6 +63,7 @@ url='https://github.com/ledatelescope/bifrost', packages=find_packages(), scripts=scripts, + python_requires='>=3.6', install_requires=[ "numpy>=1.8.1", "contextlib2>=0.4.0", From a8dbc15d016207e651e0dd2d8571264eed0ea212 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 13:56:32 -0600 Subject: [PATCH 47/56] More f strings. --- python/bifrost/block.py | 6 +++--- python/bifrost/map.py | 10 +++++----- python/bifrost/ndarray.py | 5 ++--- python/bifrost/pipeline.py | 5 ++--- python/bifrost/sigproc2.py | 4 ++-- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/python/bifrost/block.py b/python/bifrost/block.py index 99c4d916e..16db2cc44 100644 --- a/python/bifrost/block.py +++ b/python/bifrost/block.py @@ -1010,7 +1010,7 @@ def __init__(self, generator, outputs=1, grab_headers=False, changing=True): equal to the number of outgoing rings attached to this block. @param[in] changing Whether or not the arrays will be different in shape""" super(NumpySourceBlock, self).__init__() - outputs = ['out_%d' % (i + 1) for i in range(outputs)] + outputs = [f"out_{i+1}" for i in range(outputs)] self.ring_names = {} for output_name in outputs: ring_description = "Output number " + output_name[4:] @@ -1026,7 +1026,7 @@ def calculate_output_settings(self, arrays): @param[in] arrays The arrays outputted by self.generator""" for index in range(len(self.ring_names)): assert isinstance(arrays[index], np.ndarray) - ring_name = 'out_%d' % (index + 1) + ring_name = f"out_{index+1}" self.header[ring_name] = { 'dtype': str(arrays[index].dtype), 'shape': list(arrays[index].shape), @@ -1038,7 +1038,7 @@ def load_user_headers(self, headers, arrays): @param[in] headers List of dictionaries from self.generator for each ring's sequence header""" for i, header in enumerate(headers): - ring_name = 'out_%d' % (i + 1) + ring_name = f"out_{i+1}" for parameter in header: self.header[ring_name][parameter] = header[parameter] if 'dtype' in header: diff --git a/python/bifrost/map.py b/python/bifrost/map.py index cb11189c3..4a50d1b2f 100644 --- a/python/bifrost/map.py +++ b/python/bifrost/map.py @@ -152,16 +152,16 @@ def list_map_cache() -> None: version = fh.read() mapcache, runtime, driver = version.split(None, 2) mapcache = int(mapcache, 10) - mapcache = "%i.%i" % (mapcache//1000, (mapcache//10) % 1000) + mapcache = f"{mapcache//1000}.{mapcache//10) % 1000}" runtime = int(runtime, 10) - runtime = "%i.%i" % (runtime//1000, (runtime//10) % 1000) + runtime = f"{runtime//1000}.{runtime//10) % 1000}" driver = int(driver, 10) - driver = "%i.%i" % (driver//1000, (driver//10) % 1000) + driver = f"{driver//1000}.{driver//10) % 1000}" entries = glob.glob(os.path.join(cache_path, '*.inf')) - output += "\nCache version: %s (map cache) %s (runtime), %s (driver)" % (mapcache, runtime, driver) - output += "\nCache entries: %i" % len(entries) + output += f"\nCache version: {mapcache} (map cache) {runtime} (runtime), {driver} (driver)" + output += f"\nCache entries: {len(entries)}" except OSError: pass diff --git a/python/bifrost/ndarray.py b/python/bifrost/ndarray.py index 98f9da658..ba9a88385 100644 --- a/python/bifrost/ndarray.py +++ b/python/bifrost/ndarray.py @@ -204,9 +204,8 @@ def __new__(cls, base=None, space=None, shape=None, dtype=None, base = base.astype(dtype.as_numpy_dtype()) base = ndarray(base) # View base as bf.ndarray if dtype is not None and base.bf.dtype != dtype: - raise TypeError('Unable to convert type %s to %s during ' - 'array construction' % - (base.bf.dtype, dtype)) + raise TypeError(f"Unable to convert type {base.bf.dtype} to {dtype} during " + "array construction") #base = base.view(cls #if dtype is not None: # base = base.astype(DataType(dtype).as_numpy_dtype()) diff --git a/python/bifrost/pipeline.py b/python/bifrost/pipeline.py index 9058b0b67..c8d14a6fc 100644 --- a/python/bifrost/pipeline.py +++ b/python/bifrost/pipeline.py @@ -282,7 +282,7 @@ def _handle_signal_shutdown(self, signum, frame): reversed(sorted(signal.__dict__.items())) if v.startswith('SIG') and not v.startswith('SIG_')) - warnings.warn("Received signal %i %s, shutting down pipeline" % (signum, SIGNAL_NAMES[signum]), RuntimeWarning) + warnings.warn(f"Received signal {signum} {SIGNAL_NAMES[signum]}, shutting down pipeline", RuntimeWarning) self.shutdown() def __enter__(self): thread_local.pipeline_stack.append(self) @@ -340,8 +340,7 @@ def __init__(self, irings: List[Union["Block",Ring]], valid_inp_spaces = self._define_valid_input_spaces() for i, (iring, valid_spaces) in enumerate(zip(irings, valid_inp_spaces)): if not memory.space_accessible(iring.space, valid_spaces): - raise ValueError("Block %s input %i's space must be accessible from one of: %s" % - (self.name, i, str(valid_spaces))) + raise ValueError(f"Block {self.name} input {i}'s space must be accessible from one of: {valid_spaces}") self.orings = [] # Update this in subclass constructors self.shutdown_event = threading.Event() self.bind_proclog = ProcLog(self.name + "/bind") diff --git a/python/bifrost/sigproc2.py b/python/bifrost/sigproc2.py index f2b07086f..46bae42a8 100644 --- a/python/bifrost/sigproc2.py +++ b/python/bifrost/sigproc2.py @@ -337,7 +337,7 @@ def read(self, nframe_or_start: int, end: Optional[int]=None) -> np.ndarray: start = nframe_or_start or 0 if start * self.frame_size * self.nbit % 8 != 0: raise ValueError("Start index must be aligned with byte boundary " + - "(idx=%i, nbit=%i)" % (start, self.nbit)) + f"(idx={start}, nbit={self.nbit})") self.seek(start * self.frame_size * self.nbit // 8) if end == -1: end = self.nframe() @@ -347,7 +347,7 @@ def read(self, nframe_or_start: int, end: Optional[int]=None) -> np.ndarray: if self.nbit < 8: if nframe * self.frame_size * self.nbit % 8 != 0: raise ValueError("No. frames must correspond to whole number of bytes " + - "(idx=%i, nbit=%i)" % (nframe, self.nbit)) + f"(idx={nframe}, nbit={self.nbit})") #data = np.fromfile(self.f, np.uint8, # nframe * self.frame_size * self.nbit // 8) #requested_nbyte = nframe * self.frame_nbyte From 848a8d0e7b3336f4b135271554297a2ed425b745 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 13:58:27 -0600 Subject: [PATCH 48/56] Preserve spacing. --- python/bifrost/block.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/bifrost/block.py b/python/bifrost/block.py index 16db2cc44..ce61b6935 100644 --- a/python/bifrost/block.py +++ b/python/bifrost/block.py @@ -1010,7 +1010,7 @@ def __init__(self, generator, outputs=1, grab_headers=False, changing=True): equal to the number of outgoing rings attached to this block. @param[in] changing Whether or not the arrays will be different in shape""" super(NumpySourceBlock, self).__init__() - outputs = [f"out_{i+1}" for i in range(outputs)] + outputs = [f"out_{i + 1}" for i in range(outputs)] self.ring_names = {} for output_name in outputs: ring_description = "Output number " + output_name[4:] @@ -1026,7 +1026,7 @@ def calculate_output_settings(self, arrays): @param[in] arrays The arrays outputted by self.generator""" for index in range(len(self.ring_names)): assert isinstance(arrays[index], np.ndarray) - ring_name = f"out_{index+1}" + ring_name = f"out_{index + 1}" self.header[ring_name] = { 'dtype': str(arrays[index].dtype), 'shape': list(arrays[index].shape), @@ -1038,7 +1038,7 @@ def load_user_headers(self, headers, arrays): @param[in] headers List of dictionaries from self.generator for each ring's sequence header""" for i, header in enumerate(headers): - ring_name = f"out_{i+1}" + ring_name = f"out_{i + 1}" for parameter in header: self.header[ring_name][parameter] = header[parameter] if 'dtype' in header: From 808422d77589d6284f8f4005d291ade6ed319671 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:04:26 -0600 Subject: [PATCH 49/56] A few more f strings. --- python/bifrost/views/basic_views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/bifrost/views/basic_views.py b/python/bifrost/views/basic_views.py index d67e17a89..2e525af53 100644 --- a/python/bifrost/views/basic_views.py +++ b/python/bifrost/views/basic_views.py @@ -114,13 +114,12 @@ def header_transform(hdr, axis=axis): tensor = hdr['_tensor'] specified_axis = axis if isinstance(axis, str): - specified_axis = "'%s'" % specified_axis + specified_axis = f"'{specified_axis}'" axis = tensor['labels'].index(axis) if axis < 0: axis += len(tensor['shape']) + 1 if tensor['shape'][axis] != 1: - raise ValueError("Cannot delete non-unitary axis %s with shape %i" - % (specified_axis, tensor['shape'][axis])) + raise ValueError(f"Cannot delete non-unitary axis {specified_axis} with shape {tensor['shape'][axis]}") del tensor['shape'][axis] if 'labels' in tensor: del tensor['labels'][axis] @@ -160,8 +159,7 @@ def header_transform(hdr, axis=axis, n=n, label=label): else: # Axis is not frame axis if shape[axis] % n: - raise ValueError("Split does not evenly divide axis (%i // %i)" % - (tensor['shape'][axis], n)) + raise ValueError(f"Split does not evenly divide axis ({tensor['shape'][axis]} // {n})") shape[axis] //= n shape.insert(axis + 1, n) if 'units' in tensor: @@ -205,7 +203,7 @@ def header_transform(hdr, axis1=axis1, axis2=axis2, label=label): scale2 = convert_units(scale2, units2, units1) if not isclose(scale1, n * scale2): raise ValueError("Scales of merge axes do not line up: " - "%f != %f" % (scale1, n * scale2)) + f"{scale1} != {n * scale2}") tensor['scales'][axis1][1] = scale2 del tensor['scales'][axis2] del tensor['units'][axis2] From e01355f2b95b8be3101a11620e4800a707453b60 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:12:57 -0600 Subject: [PATCH 50/56] Add a Python version check to configure as well. --- configure | 14 +++++++++++++- configure.ac | 7 ++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 0ad981380..d71011c7d 100755 --- a/configure +++ b/configure @@ -22310,7 +22310,19 @@ fi fi if test x${PYTHON} != xno then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $PYTHON as ctypesgen" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if $PYTHON is version 3.6 or later" >&5 +printf %s "checking if $PYTHON is version 3.6 or later... " >&6; } + if ! ${PYTHON} -c "import sys; assert(sys.version_info >= (3,6,0))" 2>/dev/null +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: python module will not be built" >&5 +printf "%s\n" "$as_me: WARNING: python module will not be built" >&2;} +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $PYTHON as ctypesgen" >&5 printf %s "checking whether $PYTHON as ctypesgen... " >&6; } if ! ${PYTHON} -c "import ctypesgen" 2>/dev/null then : diff --git a/configure.ac b/configure.ac index 9aebfd263..50f6ebe88 100644 --- a/configure.ac +++ b/configure.ac @@ -232,7 +232,12 @@ AS_IF([test x$enable_python != xno], AS_IF([test x${PYTHON3} != xno], [AC_SUBST([PYTHON], [$PYTHON3])])]) AS_IF([test x${PYTHON} != xno], - [AC_MSG_CHECKING([whether $PYTHON as ctypesgen]) + [AC_MSG_CHECKING([if $PYTHON is version 3.6 or later]) + AS_IF([! ${PYTHON} -c "import sys; assert(sys.version_info >= (3,6,0))" 2>/dev/null], + [AC_MSG_RESULT([no]) + AC_MSG_WARN([python module will not be built])], + [AC_MSG_RESULT([yes])]) + AC_MSG_CHECKING([whether $PYTHON as ctypesgen]) AS_IF([! ${PYTHON} -c "import ctypesgen" 2>/dev/null], [AC_MSG_RESULT([no]) AC_MSG_WARN([python module will not be built])], From 82cd44823f805d4ade59455c791270a010bb8660 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:23:16 -0600 Subject: [PATCH 51/56] a or b can by None. --- python/bifrost/linalg.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/bifrost/linalg.py b/python/bifrost/linalg.py index 368e33049..339f3deb9 100644 --- a/python/bifrost/linalg.py +++ b/python/bifrost/linalg.py @@ -25,6 +25,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from typing import Optional + from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray from bifrost.ndarray import ndarray @@ -35,7 +37,7 @@ class LinAlg(BifrostObject): def __init__(self): BifrostObject.__init__(self, _bf.bfLinAlgCreate, _bf.bfLinAlgDestroy) - def matmul(self, alpha: float, a: ndarray, b: ndarray, beta: float, c: ndarray) -> ndarray: + def matmul(self, alpha: float, a: Optional[ndarray], b: Optional[ndarray], beta: float, c: ndarray) -> ndarray: """Computes: c = alpha*a.b + beta*c or if b is None: From 523c8a4794617334aa7c7539b718ae50242f30bf Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:24:02 -0600 Subject: [PATCH 52/56] Reorder imports. --- python/bifrost/linalg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/linalg.py b/python/bifrost/linalg.py index 339f3deb9..3c7ffc40d 100644 --- a/python/bifrost/linalg.py +++ b/python/bifrost/linalg.py @@ -25,12 +25,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import Optional - from bifrost.libbifrost import _bf, _check, BifrostObject from bifrost.ndarray import asarray from bifrost.ndarray import ndarray +from typing import Optional + from bifrost import telemetry telemetry.track_module() From 5c4bdb1584a36508c3e3b486e32f01086f9b9a34 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:25:08 -0600 Subject: [PATCH 53/56] Drop encoding line. --- python/bifrost/udp_transmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/bifrost/udp_transmit.py b/python/bifrost/udp_transmit.py index f7bed1b71..f4f183c6f 100644 --- a/python/bifrost/udp_transmit.py +++ b/python/bifrost/udp_transmit.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- + # Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions From 12313693ce55de97c1e11f278ea43640adb225be Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Fri, 25 Aug 2023 14:41:37 -0600 Subject: [PATCH 54/56] Ugh. --- python/bifrost/map.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/bifrost/map.py b/python/bifrost/map.py index 4a50d1b2f..8f3988f71 100644 --- a/python/bifrost/map.py +++ b/python/bifrost/map.py @@ -152,11 +152,11 @@ def list_map_cache() -> None: version = fh.read() mapcache, runtime, driver = version.split(None, 2) mapcache = int(mapcache, 10) - mapcache = f"{mapcache//1000}.{mapcache//10) % 1000}" + mapcache = f"{mapcache//1000}.{(mapcache//10) % 1000}" runtime = int(runtime, 10) - runtime = f"{runtime//1000}.{runtime//10) % 1000}" + runtime = f"{runtime//1000}.{(runtime//10) % 1000}" driver = int(driver, 10) - driver = f"{driver//1000}.{driver//10) % 1000}" + driver = f"{driver//1000}.{(driver//10) % 1000}" entries = glob.glob(os.path.join(cache_path, '*.inf')) From ff06f4539f41f9b9d07169fe9b7af2519ffb9cb2 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 28 Aug 2023 16:10:15 -0600 Subject: [PATCH 55/56] Only check for ctypesgen if we have an allowed Python version. --- configure | 7 ++++++- configure.ac | 8 +++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/configure b/configure index d71011c7d..b94108dcb 100755 --- a/configure +++ b/configure @@ -22318,11 +22318,16 @@ then : printf "%s\n" "no" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: python module will not be built" >&5 printf "%s\n" "$as_me: WARNING: python module will not be built" >&2;} + PYTHON=no + else $as_nop { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $PYTHON as ctypesgen" >&5 +fi + if test x${PYTHON} != xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $PYTHON as ctypesgen" >&5 printf %s "checking whether $PYTHON as ctypesgen... " >&6; } if ! ${PYTHON} -c "import ctypesgen" 2>/dev/null then : diff --git a/configure.ac b/configure.ac index 50f6ebe88..062f52bca 100644 --- a/configure.ac +++ b/configure.ac @@ -235,9 +235,11 @@ AS_IF([test x$enable_python != xno], [AC_MSG_CHECKING([if $PYTHON is version 3.6 or later]) AS_IF([! ${PYTHON} -c "import sys; assert(sys.version_info >= (3,6,0))" 2>/dev/null], [AC_MSG_RESULT([no]) - AC_MSG_WARN([python module will not be built])], - [AC_MSG_RESULT([yes])]) - AC_MSG_CHECKING([whether $PYTHON as ctypesgen]) + AC_MSG_WARN([python module will not be built]) + AC_SUBST([PYTHON], [no])], + [AC_MSG_RESULT([yes])])]) + AS_IF([test x${PYTHON} != xno], + [AC_MSG_CHECKING([whether $PYTHON as ctypesgen]) AS_IF([! ${PYTHON} -c "import ctypesgen" 2>/dev/null], [AC_MSG_RESULT([no]) AC_MSG_WARN([python module will not be built])], From f5b0b02cb1cc8cd446a3f4d90cfc824276c56b64 Mon Sep 17 00:00:00 2001 From: jaycedowell Date: Mon, 28 Aug 2023 16:17:15 -0600 Subject: [PATCH 56/56] Drop shell=True. --- tools/pipeline2dot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipeline2dot.py b/tools/pipeline2dot.py index 4c436387d..b9652796a 100755 --- a/tools/pipeline2dot.py +++ b/tools/pipeline2dot.py @@ -62,7 +62,7 @@ def get_process_details(pid): data = {'user':'', 'cpu':0.0, 'mem':0.0, 'etime':'00:00', 'threads':0} try: - output = subprocess.check_output(f"ps o user,pcpu,pmem,etime,nlwp {pid}", shell=True) + output = subprocess.check_output(['ps', 'o', 'user,pcpu,pmem,etime,nlwp', str(pid)]) output = output.decode() output = output.split('\n')[1] fields = output.split(None, 4)