moved to python 2.12

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10 AS hsds-base
+FROM python:3.12 AS hsds-base
 # FROM hdfgroup/hdf5lib:1.14.0 as hsds-base
 
 # Install Curl

Pipfile

@@ -9,6 +9,7 @@ aiobotocore = "==2.5.0"
 aiohttp-cors = "*"
 aiofiles = "*"
 azure-storage-blob = "*"
+bitshuffle = "git+https://github.com/kiyo-masui/bitshuffle"
 botocore = "*"
 cryptography = "*"
 h5py = ">=3.6.0"

hsds/config.py

@@ -12,7 +12,7 @@
 import os
 import sys
 import yaml
-from pkg_resources import resource_filename
+from importlib_resources import files
 
 cfg = {}
 
@@ -99,10 +99,11 @@ def _load_cfg():
             break
     if not yml_file:
         # use yaml file embedded in package
-        yml_file = resource_filename("admin", "config/config.yml")
+        yml_file = files('admin.config').joinpath('config.yml')
 
         if not yml_file:
             raise FileNotFoundError("unable to load config.yml")
+    #
     debug(f"_load_cfg with '{yml_file}'")
     try:
         with open(yml_file, "r") as f:

hsds/util/storUtil.py

@@ -19,7 +19,7 @@
 import zlib
 import numpy as np
 import numcodecs as codecs
-# import bitshuffle
+import bitshuffle
 from json import JSONDecodeError
 from aiohttp.web_exceptions import HTTPInternalServerError
 
@@ -104,16 +104,12 @@ def _shuffle(codec, data, chunk_shape=None, dtype=None):
         # bit shuffle, use bitshuffle package
         # bitshufle is expecting numpy array
         # todo - enable block size to be set as part of the filter options
-        log.error("bitshuffle codec is not currently supportd")
-        raise ValueError()
-        """
         block_size = config.get("bit_shuffle_default_blocksize", default=2048)
 
         data = np.frombuffer(data, dtype=dtype)
         data = data.reshape(chunk_shape)
         log.debug(f"bitshuffle.compress_lz4 - chunk_size: {chunk_size} block_size: {block_size}")
         arr = bitshuffle.compress_lz4(data, block_size)
-        """
 
     else:
         log.error(f"Unexpected codec: {codec} for _shuffle")
@@ -139,7 +135,7 @@ def _shuffle(codec, data, chunk_shape=None, dtype=None):
 
 def _unshuffle(codec, data, dtype=None, chunk_shape=None):
     item_size = dtype.itemsize
-    # chunk_size = int(np.prod(chunk_shape)) * item_size
+    chunk_size = int(np.prod(chunk_shape)) * item_size
 
     if codec == 1:
         # byte shuffle, use numcodecs Shuffle
@@ -148,9 +144,6 @@ def _unshuffle(codec, data, dtype=None, chunk_shape=None):
     elif codec == 2:
         # bit shuffle, use bitshuffle
         # bitshufle is expecting numpy array
-        log.error("bitshuffle codec currently unsupported")
-        raise ValueError()
-        """
         data = np.frombuffer(data, dtype=np.dtype("uint8"))
         if len(data) < 12:
             # there should be at least 12 bytes for the header
@@ -179,7 +172,6 @@ def _unshuffle(codec, data, dtype=None, chunk_shape=None):
         except Exception as e:
             log.error(f"except using bitshuffle.decompress_lz4: {e}")
             raise HTTPInternalServerError()
-        """
     else:
         log.error(f"Unexpected codec: {codec} for _shuffle")
         raise ValueError()

pyproject.toml

@@ -39,9 +39,10 @@ dependencies = [
     "aiohttp_cors",
     "aiofiles",
     "azure-storage-blob",
-    # "bitshuffle",
+    "bitshuffle@git+https://github.com/kiyo-masui/bitshuffle",
     "cryptography",
     "h5py >= 3.6.0",
+    "importlib_resources",
     "numcodecs",
     "numpy",
     "psutil",

tests/unit/shuffle_test.py

@@ -41,8 +41,6 @@ def testByteShuffle(self):
             self.assertEqual(data[i], unshuffled[i])
 
     def testBitShuffle(self):
-        print("skipping bitshuffle test")
-        return
         arr = np.array(list(range(100)), dtype="<u2")
         data = arr.tobytes()
         # Bit Shuffle