Merge branch 'master' into feat/GPU

.github/workflows/cmake.yml

@@ -13,7 +13,7 @@ jobs:
     # cross-platform coverage.
     # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#configuring-a-build-matrix
     runs-on: ubuntu-latest
-    container: philipdavis/dspaces:bionic-margo-plus
+    container: philipdavis/dspaces-ci:v20082024
 
     defaults:
       run:
@@ -27,19 +27,20 @@ jobs:
       # We'll use this as our working directory for all subsequent commands
       run: mkdir -p ${{runner.workspace}}/build
 
+    - name: Extend python path
+      run: sed -i '1 a export PYTHONPATH=\/dspaces\/modules' /entrypoint.sh
+
     - name: Configure CMake
       # Note the current convention is to use the -S and -B options here to specify source 
       # and build directories, but this is only available with CMake 3.13 and higher.  
       # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_TESTS=true -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpifort
+      run: /entrypoint.sh cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_TESTS=true -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpifort
 
     - name: Build
       # Execute the build.  You can specify a specific target with "--target <NAME>"
-      run: cmake --build . --config $BUILD_TYPE
+      run: /entrypoint.sh cmake --build . --config $BUILD_TYPE
 
     - name: Test
       # Execute tests defined by the CMake configuration.  
       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
-      run: ctest -V -C $BUILD_TYPE
-      #env:
-      #  DSPACES_DEBUG: 1
+      run: /entrypoint.sh ctest -V -C $BUILD_TYPE

.github/workflows/format.yml

@@ -6,7 +6,7 @@ on: [push, pull_request]
 jobs:
     format:
         runs-on: ubuntu-latest
-        container: philipdavis/dspaces:bionic-margo-plus             
+        container: philipdavis/dspaces-deps:v23052024 
 
         steps:
         - uses: actions/checkout@v2

CMakeLists.txt

@@ -65,13 +65,19 @@ xpkg_import_module (liblz4 REQUIRED liblz4)
 include(python_bindings)
 
 find_package(MPI COMPONENTS REQUIRED)
+
+set(DSPACES_USE_OPENMP ON CACHE STRING "Use OpenMP for operations")
+mark_as_advanced(DSPACES_USE_OPENMP)
 find_package(OpenMP)
-if(OPENMP_FOUND)
+if(OPENMP_FOUND AND DSPACES_USE_OPENMP)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
     add_definitions(-DOPS_USE_OPENMP)
 endif()
-find_package(CURL REQUIRED) 
+find_package(CURL)
+if(CURL_FOUND)
+    add_definitions(-DDSPACES_HAVE_CURL)
+endif()
 
 include(CheckLanguage)
 check_language(Fortran)

bindings/python/dspaces.py

@@ -2,6 +2,8 @@
 from dspaces.dspaces_wrapper import *
 import numpy as np
 import dill as pickle
+from inspect import signature
+import json
 
 @dataclass
 class DSObject:
@@ -10,6 +12,31 @@ class DSObject:
     lb: tuple[int, ...]
     ub: tuple[int, ...]
 
+    def toReq(self, nspace):
+        return({
+            'var_name': (nspace + self.name).encode('ascii'),
+            'ver': self.version,
+            'lb': self.lb,
+            'ub': self.ub
+        })
+
+@dataclass
+class DSRegHandle:
+    namespace: str
+    parameters: dict
+
+class DSModuleError(Exception):
+    def __init__(self, errno):
+        self.errno = errno
+
+class DSRemoteFaultError(Exception):
+    def __init__(self, errno):
+        self.errno = errno
+
+class DSConnectionError(Exception):
+    def __init__(self, errno):
+        self.errno = errno
+
 class DSServer:
     def __init__(self, conn = "sockets", comm = None, conf = "dataspaces.conf"):
         from mpi4py import MPI
@@ -42,7 +69,8 @@ def __init__(self, comm = None, conn = None, rank = None):
                 self.client = wrapper_dspaces_init_wan(listen_str.encode('ascii'), conn.encode('ascii'), rank)
 
     def __del__(self):
-        wrapper_dspaces_fini(self.client) 
+        if hasattr(self, 'client'):
+            wrapper_dspaces_fini(self.client) 
 
     def KillServer(self, token_count = 1):
         for token in range(token_count):
@@ -63,7 +91,49 @@ def Get(self, name, version, lb, ub, timeout, dtype = None):
         return wrapper_dspaces_get(self.client, (self.nspace + name).encode('ascii'), version, lb, ub, passed_type, timeout)    
 
     def Exec(self, name, version, lb=None, ub=None, fn=None):
-        res = wrapper_dspaces_pexec(self.client, (self.nspace + name).encode('ascii'), version, lb, ub, pickle.dumps(fn), fn.__name__.encode('ascii'))
+        arg = DSObject(name=name, version=version, lb=lb, ub=ub)
+        return(self.VecExec([arg], fn))
+
+    def Register(self, type, name, data):
+        nspace, reg_id = wrapper_dspaces_register(self.client,
+                                        type.encode('ascii'),
+                                        name.encode('ascii'),
+                                        json.dumps(data).encode('ascii'))
+        if reg_id < 0:
+            if reg_id == -3:
+                raise DSModuleError(reg_id)
+            elif reg_id == -2 or reg_id == -1 or reg_id == -4 or reg_id == -5:
+                raise DSRemoteFaultError(reg_id)
+            elif reg_id == -6:
+                raise DSConnectionError(reg_id)
+            else:
+                raise Exception("unknown failure")
+        return(DSRegHandle(nspace, {'id':str(reg_id)}))
+
+    def VecExec(self, objs:list[DSObject] = [], fn=None):
+        if objs and not hasattr(objs, '__iter__'):
+            raise TypeError('reqs should be a list of arguments, if present')
+        if fn:
+            if objs:
+                test_args = [np.arange(1) for x in objs]
+            else:
+                test_args = []
+            sig = signature(fn)
+            try:
+                sig.bind(*test_args)
+            except:
+                print("Mismatch between fn arguments and number of reqs.")
+                return
+            fn_ser = pickle.dumps(fn)
+            fn_name = fn.__name__.encode('ascii')
+        else:
+            fn_ser = None
+            fn_name = None
+        if objs:
+            args = [obj.toReq(self.nspace) for obj in objs]
+        else:
+            args = []
+        res = wrapper_dspaces_pexec(self.client, args, fn_ser, fn_name)
         if res:
             return pickle.loads(res)
         else:
@@ -75,7 +145,7 @@ def DefineGDim(self, name, gdim):
     def GetVars(self):
         return wrapper_dspaces_get_vars(self.client)
 
-    def GetObjVars(self, var_name):
+    def GetVarObjs(self, var_name):
         ret_objs = []
         wrapper_results =  wrapper_dspaces_get_var_objs(self.client, var_name.encode('ascii'))
         for obj in wrapper_results:
@@ -142,6 +212,7 @@ def arctan(self):
         return(obj)
     def exec(self):
         return(wrapper_dspaces_ops_calc(self.client.client, self.expr))
+
 class DSConst(DSExpr):
     def __init__(self, client, val):
         DSExpr.__init__(self, client)