diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5bdaf460..33d90ec5 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -13,7 +13,7 @@ jobs: # cross-platform coverage. # See: https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#configuring-a-build-matrix runs-on: ubuntu-latest - container: philipdavis/dspaces:bionic-margo-plus + container: philipdavis/dspaces-ci:v20082024 defaults: run: @@ -27,19 +27,20 @@ jobs: # We'll use this as our working directory for all subsequent commands run: mkdir -p ${{runner.workspace}}/build + - name: Extend python path + run: sed -i '1 a export PYTHONPATH=\/dspaces\/modules' /entrypoint.sh + - name: Configure CMake # Note the current convention is to use the -S and -B options here to specify source # and build directories, but this is only available with CMake 3.13 and higher. # The CMake binaries on the Github Actions machines are (as of this writing) 3.12 - run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_TESTS=true -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpifort + run: /entrypoint.sh cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DENABLE_TESTS=true -DCMAKE_C_COMPILER=mpicc -DCMAKE_Fortran_COMPILER=mpifort - name: Build # Execute the build. You can specify a specific target with "--target " - run: cmake --build . --config $BUILD_TYPE + run: /entrypoint.sh cmake --build . --config $BUILD_TYPE - name: Test # Execute tests defined by the CMake configuration. # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail - run: ctest -V -C $BUILD_TYPE - #env: - # DSPACES_DEBUG: 1 + run: /entrypoint.sh ctest -V -C $BUILD_TYPE diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e799c9c1..79a9b171 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -6,7 +6,7 @@ on: [push, pull_request] jobs: format: runs-on: ubuntu-latest - container: philipdavis/dspaces:bionic-margo-plus + container: philipdavis/dspaces-deps:v23052024 steps: - uses: actions/checkout@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 969c0588..9a6b95cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,13 +65,19 @@ xpkg_import_module (liblz4 REQUIRED liblz4) include(python_bindings) find_package(MPI COMPONENTS REQUIRED) + +set(DSPACES_USE_OPENMP ON CACHE STRING "Use OpenMP for operations") +mark_as_advanced(DSPACES_USE_OPENMP) find_package(OpenMP) -if(OPENMP_FOUND) +if(OPENMP_FOUND AND DSPACES_USE_OPENMP) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") add_definitions(-DOPS_USE_OPENMP) endif() -find_package(CURL REQUIRED) +find_package(CURL) +if(CURL_FOUND) + add_definitions(-DDSPACES_HAVE_CURL) +endif() include(CheckLanguage) check_language(Fortran) diff --git a/bindings/python/dspaces.py b/bindings/python/dspaces.py index d7119ccc..2f38a8ac 100644 --- a/bindings/python/dspaces.py +++ b/bindings/python/dspaces.py @@ -2,6 +2,8 @@ from dspaces.dspaces_wrapper import * import numpy as np import dill as pickle +from inspect import signature +import json @dataclass class DSObject: @@ -10,6 +12,31 @@ class DSObject: lb: tuple[int, ...] ub: tuple[int, ...] + def toReq(self, nspace): + return({ + 'var_name': (nspace + self.name).encode('ascii'), + 'ver': self.version, + 'lb': self.lb, + 'ub': self.ub + }) + +@dataclass +class DSRegHandle: + namespace: str + parameters: dict + +class DSModuleError(Exception): + def __init__(self, errno): + self.errno = errno + +class DSRemoteFaultError(Exception): + def __init__(self, errno): + self.errno = errno + +class DSConnectionError(Exception): + def __init__(self, errno): + self.errno = errno + class DSServer: def __init__(self, conn = "sockets", comm = None, conf = "dataspaces.conf"): from mpi4py import MPI @@ -42,7 +69,8 @@ def __init__(self, comm = None, conn = None, rank = None): self.client = wrapper_dspaces_init_wan(listen_str.encode('ascii'), conn.encode('ascii'), rank) def __del__(self): - wrapper_dspaces_fini(self.client) + if hasattr(self, 'client'): + wrapper_dspaces_fini(self.client) def KillServer(self, token_count = 1): for token in range(token_count): @@ -63,7 +91,49 @@ def Get(self, name, version, lb, ub, timeout, dtype = None): return wrapper_dspaces_get(self.client, (self.nspace + name).encode('ascii'), version, lb, ub, passed_type, timeout) def Exec(self, name, version, lb=None, ub=None, fn=None): - res = wrapper_dspaces_pexec(self.client, (self.nspace + name).encode('ascii'), version, lb, ub, pickle.dumps(fn), fn.__name__.encode('ascii')) + arg = DSObject(name=name, version=version, lb=lb, ub=ub) + return(self.VecExec([arg], fn)) + + def Register(self, type, name, data): + nspace, reg_id = wrapper_dspaces_register(self.client, + type.encode('ascii'), + name.encode('ascii'), + json.dumps(data).encode('ascii')) + if reg_id < 0: + if reg_id == -3: + raise DSModuleError(reg_id) + elif reg_id == -2 or reg_id == -1 or reg_id == -4 or reg_id == -5: + raise DSRemoteFaultError(reg_id) + elif reg_id == -6: + raise DSConnectionError(reg_id) + else: + raise Exception("unknown failure") + return(DSRegHandle(nspace, {'id':str(reg_id)})) + + def VecExec(self, objs:list[DSObject] = [], fn=None): + if objs and not hasattr(objs, '__iter__'): + raise TypeError('reqs should be a list of arguments, if present') + if fn: + if objs: + test_args = [np.arange(1) for x in objs] + else: + test_args = [] + sig = signature(fn) + try: + sig.bind(*test_args) + except: + print("Mismatch between fn arguments and number of reqs.") + return + fn_ser = pickle.dumps(fn) + fn_name = fn.__name__.encode('ascii') + else: + fn_ser = None + fn_name = None + if objs: + args = [obj.toReq(self.nspace) for obj in objs] + else: + args = [] + res = wrapper_dspaces_pexec(self.client, args, fn_ser, fn_name) if res: return pickle.loads(res) else: @@ -75,7 +145,7 @@ def DefineGDim(self, name, gdim): def GetVars(self): return wrapper_dspaces_get_vars(self.client) - def GetObjVars(self, var_name): + def GetVarObjs(self, var_name): ret_objs = [] wrapper_results = wrapper_dspaces_get_var_objs(self.client, var_name.encode('ascii')) for obj in wrapper_results: @@ -142,6 +212,7 @@ def arctan(self): return(obj) def exec(self): return(wrapper_dspaces_ops_calc(self.client.client, self.expr)) + class DSConst(DSExpr): def __init__(self, client, val): DSExpr.__init__(self, client) diff --git a/bindings/python/dspaces_wrapper.c b/bindings/python/dspaces_wrapper.c index d65aacca..8a3ea47f 100644 --- a/bindings/python/dspaces_wrapper.c +++ b/bindings/python/dspaces_wrapper.c @@ -23,11 +23,12 @@ PyObject *wrapper_dspaces_init(int rank) clientp = malloc(sizeof(*clientp)); // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = dspaces_init(rank, clientp); - //Py_END_ALLOW_THREADS - // clang-format on - if(ret != 0) { + Py_END_ALLOW_THREADS + // clang-format on + if(ret != 0) + { sprintf(err_str, "dspaces_init() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; @@ -55,11 +56,12 @@ PyObject *wrapper_dspaces_init_mpi(PyObject *commpy) clientp = malloc(sizeof(*clientp)); // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_mpi(*comm_p, clientp); - //Py_END_ALLOW_THREADS - // clang-format on - if(ret != 0) { + Py_END_ALLOW_THREADS + // clang-format on + if(ret != 0) + { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; @@ -83,11 +85,12 @@ PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, clientp = malloc(sizeof(*clientp)); // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan(listen_str, conn, rank, clientp); - //Py_END_ALLOW_THREADS - // clang-format on - if(ret != 0) { + Py_END_ALLOW_THREADS + // clang-format on + if(ret != 0) + { sprintf(err_str, "dspaces_init_wan() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; @@ -116,11 +119,12 @@ PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, clientp = malloc(sizeof(*clientp)); // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan_mpi(listen_str, conn, *comm_p, clientp); - //Py_END_ALLOW_THREADS - // clang-format on - if(ret != 0) { + Py_END_ALLOW_THREADS + // clang-format on + if(ret != 0) + { sprintf(err_str, "dspaces_init_wan_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; @@ -149,11 +153,12 @@ PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, serverp = malloc(sizeof(*serverp)); // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = dspaces_server_init(listen_str, *comm_p, conf, serverp); - //Py_END_ALLOW_THREADS - // clang-format on - if(ret != 0) { + Py_END_ALLOW_THREADS + // clang-format on + if(ret != 0) + { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; @@ -204,20 +209,20 @@ void wrapper_dspaces_put(PyObject *clientppy, PyObject *obj, const char *name, PyObject *item; int i; - //Reverse order of indices + // Reverse order of indices for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(offset, i); - lb[(ndim-1) - i] = PyLong_AsLong(item); - ub[(ndim-1) - i] = lb[(ndim-1) - i] + ((long)shape[i] - 1); + lb[(ndim - 1) - i] = PyLong_AsLong(item); + ub[(ndim - 1) - i] = lb[(ndim - 1) - i] + ((long)shape[i] - 1); } // clang-format off - //Py_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS dspaces_put_tag(*clientp, name, version, size, tag, ndim, lb, ub, data); - //Py_END_ALLOW_THREADS - // clang-format on + Py_END_ALLOW_THREADS + // clang-format on - return; + return; } PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, @@ -225,6 +230,7 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, PyObject *dtype, int timeout) { dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); + struct dspaces_req in_req = {0}, out_req = {0}; int ndim = PyTuple_GET_SIZE(lbt); uint64_t lb[ndim]; uint64_t ub[ndim]; @@ -236,36 +242,43 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, npy_intp dims[ndim]; int i; - //Reverse order of indices + // Reverse order of indices for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(lbt, i); - lb[(ndim-1) - i] = PyLong_AsLong(item); + lb[(ndim - 1) - i] = PyLong_AsLong(item); item = PyTuple_GetItem(ubt, i); - ub[(ndim-1) - i] = PyLong_AsLong(item); + ub[(ndim - 1) - i] = PyLong_AsLong(item); } + in_req.var_name = strdup(name); + in_req.ver = version; + in_req.ndim = ndim; + in_req.lb = lb; + in_req.ub = ub; // clang-format off - //Py_BEGIN_ALLOW_THREADS - dspaces_aget(*clientp, name, version, ndim, lb, ub, - &data, &tag, timeout); - //Py_END_ALLOW_THREADS - // clang-format on + Py_BEGIN_ALLOW_THREADS + dspaces_get_req(*clientp, &in_req, &out_req, timeout); + Py_END_ALLOW_THREADS + // clang-format on + data = out_req.buf; - if(data == NULL) - { + free(in_req.var_name); + + if(data == NULL) { Py_INCREF(Py_None); return (Py_None); } if(dtype == Py_None) { - descr = PyArray_DescrNewFromType(tag); + descr = PyArray_DescrNewFromType(out_req.tag); } else { descr = PyArray_DescrNew((PyArray_Descr *)dtype); } + ndim = out_req.ndim; for(i = 0; i < ndim; i++) { - dims[(ndim-1) - i] = ((ub[i] - lb[i]) + 1); + dims[(ndim - 1) - i] = ((out_req.ub[i] - out_req.lb[i]) + 1); } arr = PyArray_NewFromDescr(&PyArray_Type, descr, ndim, dims, NULL, data, 0, @@ -274,61 +287,91 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, return (arr); } -PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, - int version, PyObject *lbt, PyObject *ubt, +PyObject *wrapper_dspaces_pexec(PyObject *clientppy, PyObject *req_list, PyObject *fn, const char *fn_name) { dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); PyObject *item; - int ndim = 0; + PyObject *req_dict, *lbt, *ubt; uint64_t *lb, *ub; + int ndim; void *data; int data_size; PyObject *result; - int i; + struct dspaces_req *reqs; + int num_reqs; + int i, j; - if(lbt == Py_None || ubt == Py_None) { - if(lbt != ubt) { + num_reqs = PyList_Size(req_list); + reqs = calloc(num_reqs, sizeof(*reqs)); + + for(i = 0; i < num_reqs; i++) { + req_dict = PyList_GetItem(req_list, i); + reqs[i].var_name = strdup( + PyBytes_AsString(PyDict_GetItemString(req_dict, "var_name"))); + reqs[i].ver = PyLong_AsLong(PyDict_GetItemString(req_dict, "ver")); + lbt = PyDict_GetItemString(req_dict, "lb"); + ubt = PyDict_GetItemString(req_dict, "ub"); + + if(lbt == Py_None || ubt == Py_None) { + if(lbt != ubt) { + PyErr_SetString(PyExc_TypeError, + "both lb and ub must be set or neither"); + return (NULL); + } + } else if(PyTuple_GET_SIZE(lbt) != PyTuple_GET_SIZE(ubt)) { PyErr_SetString(PyExc_TypeError, - "both lb and ub must be set or neither"); + "lb and ub must have the same length"); return (NULL); + } else if(lbt == Py_None) { + reqs[i].ndim = 0; + } else { + reqs[i].ndim = PyTuple_GET_SIZE(lbt); } - } else if(PyTuple_GET_SIZE(lbt) != PyTuple_GET_SIZE(ubt)) { - PyErr_SetString(PyExc_TypeError, "lb and ub must have the same lenght"); - return (NULL); - } else { - ndim = PyTuple_GET_SIZE(lbt); - } - - lb = malloc(sizeof(*lb) * ndim); - ub = malloc(sizeof(*ub) * ndim); - //Reverse order of indices - for(i = 0; i < ndim; i++) { - item = PyTuple_GetItem(lbt, i); - lb[(ndim-1) - i] = PyLong_AsLong(item); - item = PyTuple_GetItem(ubt, i); - ub[(ndim-1) - i] = PyLong_AsLong(item); - } + ndim = reqs[i].ndim; + + lb = malloc(sizeof(*lb) * ndim); + ub = malloc(sizeof(*ub) * ndim); + // Reverse order of indices + for(j = 0; j < ndim; j++) { + item = PyTuple_GetItem(lbt, j); + lb[(ndim - 1) - j] = PyLong_AsLong(item); + item = PyTuple_GetItem(ubt, j); + ub[(ndim - 1) - j] = PyLong_AsLong(item); + } + reqs[i].lb = lb; + reqs[i].ub = ub; - if(!PyBytes_Check(fn)) { - PyErr_SetString(PyExc_TypeError, - "fn must be serialized as a a byte string"); - return (NULL); + if(!PyBytes_Check(fn)) { + PyErr_SetString(PyExc_TypeError, + "fn must be serialized as a a byte string"); + return (NULL); + } } - - dspaces_pexec(*clientp, name, version, ndim, lb, ub, PyBytes_AsString(fn), + // clang-format off + Py_BEGIN_ALLOW_THREADS + dspaces_mpexec(*clientp, num_reqs, reqs, PyBytes_AsString(fn), PyBytes_Size(fn) + 1, fn_name, &data, &data_size); + Py_END_ALLOW_THREADS + // clang-format on - free(lb); - free(ub); - - if(data_size > 0) { + if(data_size > 0) + { result = PyBytes_FromStringAndSize(data, data_size); - } else { + } + else + { Py_INCREF(Py_None); result = Py_None; } + for(i = 0; i < num_reqs; i++) { + free(reqs[i].var_name); + free(reqs[i].lb); + free(reqs[i].ub); + } + free(reqs); + return (result); } @@ -343,7 +386,7 @@ void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(gdimt, i); - gdim[(ndim-1) - i] = PyLong_AsLong(item); + gdim[(ndim - 1) - i] = PyLong_AsLong(item); } dspaces_define_gdim(*clientp, name, ndim, gdim); @@ -626,3 +669,25 @@ PyObject *wrapper_dspaces_ops_calc(PyObject *clientppy, PyObject *exprppy) arr = PyArray_SimpleNewFromData(ndim, array_dims, typenum, result_buf); return (arr); } + +PyObject *wrapper_dspaces_register(PyObject *clientppy, const char *type, + const char *name, const char *data) +{ + dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); + PyObject *ret; + char *nspace = NULL; + long id; + + id = dspaces_register_simple(*clientp, type, name, data, &nspace); + ret = PyTuple_New(2); + if(nspace) { + PyTuple_SET_ITEM(ret, 0, + PyUnicode_DecodeASCII(nspace, strlen(nspace), NULL)); + } else { + PyTuple_SET_ITEM(ret, 0, Py_None); + } + + PyTuple_SET_ITEM(ret, 1, PyLong_FromLong(id)); + + return (ret); +} diff --git a/bindings/python/dspaces_wrapper.h b/bindings/python/dspaces_wrapper.h index 07759202..e2c6c6cc 100644 --- a/bindings/python/dspaces_wrapper.h +++ b/bindings/python/dspaces_wrapper.h @@ -26,8 +26,7 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, int version, PyObject *lbt, PyObject *ubt, PyObject *dtype, int timeout); -PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, - int version, PyObject *lbt, PyObject *ubt, +PyObject *wrapper_dspaces_pexec(PyObject *clientppy, PyObject *req_list, PyObject *fn, const char *fn_name); void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, @@ -58,3 +57,6 @@ PyObject *wrapper_dspaces_op_new_pow(PyObject *exprppy1, PyObject *exprppy2); PyObject *wrapper_dspaces_op_new_arctan(PyObject *exprppy1); PyObject *wrapper_dspaces_ops_calc(PyObject *clientppy, PyObject *exprppy); + +PyObject *wrapper_dspaces_register(PyObject *clientppy, const char *type, + const char *name, const char *data); \ No newline at end of file diff --git a/include/bbox.h b/include/bbox.h index c432d79b..16162c2c 100644 --- a/include/bbox.h +++ b/include/bbox.h @@ -43,7 +43,7 @@ int bbox_does_intersect(const struct bbox *, const struct bbox *); void bbox_intersect(const struct bbox *, const struct bbox *, struct bbox *); int bbox_equals(const struct bbox *, const struct bbox *); -uint64_t bbox_volume(struct bbox *); +uint64_t bbox_volume(const struct bbox *); void bbox_to_intv(const struct bbox *, uint64_t, int, struct intv **, int *); void bbox_to_intv2(const struct bbox *, uint64_t, int, struct intv **, int *); void bbox_to_origin(struct bbox *, const struct bbox *); diff --git a/include/dspaces-common.h b/include/dspaces-common.h index da0e1698..20f004bf 100644 --- a/include/dspaces-common.h +++ b/include/dspaces-common.h @@ -8,6 +8,9 @@ #ifndef __DSPACES_COMMON_H #define __DSPACES_COMMON_H +#include +#include + #if defined(__cplusplus) extern "C" { #endif @@ -27,6 +30,13 @@ extern "C" { #define dspaces_ERR_CUDA -10 /* Error related to the CUDA */ #define dspaces_ERR_UTILS -11 +#define DS_MOD_EFAULT -1 +#define DS_MOD_ENODEF -2 +#define DS_MOD_ENOMOD -3 +#define DS_MOD_ENOSYS -4 +#define DS_MOD_ENOSUPPORT -5 +#define DS_MOD_ECLIENT -6 + #define DS_OBJ_RESIZE 0x02 /** @@ -36,6 +46,52 @@ extern "C" { */ static inline void ignore_result(int unused_result) { (void)unused_result; } +#define DSP_FLOAT -1 +#define DSP_INT -2 +#define DSP_LONG -3 +#define DSP_DOUBLE -4 +#define DSP_BOOL -5 +#define DSP_CHAR -6 +#define DSP_UINT -7 +#define DSP_ULONG -8 +#define DSP_BYTE -9 +#define DSP_UINT8 -10 +#define DSP_UINT16 -11 +#define DSP_UINT32 -12 +#define DSP_UINT64 -13 +#define DSP_INT8 -14 +#define DSP_INT16 -15 +#define DSP_INT32 -16 +#define DSP_INT64 -17 + +static size_t type_to_size_map[] = {0, + sizeof(float), + sizeof(int), + sizeof(long), + sizeof(double), + 1, + sizeof(char), + sizeof(unsigned), + sizeof(unsigned long), + 1, + 1, + 2, + 4, + 8, + 1, + 2, + 4, + 8}; + +static int type_to_size(int type_id) +{ + if(type_id >= 0) { + fprintf(stderr, "WARNING: type ids should be negative.\n"); + return (-1); + } + return (type_to_size_map[-type_id]); +} + #if defined(__cplusplus) } #endif diff --git a/include/dspaces-conf.h b/include/dspaces-conf.h new file mode 100644 index 00000000..7dcbf94f --- /dev/null +++ b/include/dspaces-conf.h @@ -0,0 +1,29 @@ +#ifndef __DSPACES_CONF_H__ +#define __DSPACES_CONF_H__ + +#include "bbox.h" +#include "dspaces-remote.h" +#include "list.h" + +static char *hash_strings[] = {"Dynamic", "Unitary", "SFC", "Bisection"}; + +/* Server configuration parameters */ +struct ds_conf { + int ndim; + struct coord dims; + int max_versions; + int hash_version; + int num_apps; + struct list_head *dirs; + struct remote **remotes; + int nremote; + struct list_head *mods; +}; + +int parse_conf(const char *fname, struct ds_conf *conf); + +int parse_conf_toml(const char *fname, struct ds_conf *conf); + +void print_conf(struct ds_conf *conf); + +#endif // __DSPACES_CONF_H \ No newline at end of file diff --git a/include/dspaces-logging.h b/include/dspaces-logging.h new file mode 100644 index 00000000..5c4628c2 --- /dev/null +++ b/include/dspaces-logging.h @@ -0,0 +1,36 @@ +#ifndef _DSPACES_LOGGING_H +#define _DSPACES_LOGGING_H + +#include +#include + +#define TRACE_OUT \ + do { \ + if(dspaces_trace_enabled()) { \ + ABT_unit_id tid = 0; \ + if(ABT_initialized()) \ + ABT_thread_self_id(&tid); \ + fprintf(stderr, "Rank: %i TID: %" PRIu64 " %s:%i (%s): trace\n", \ + dspaces_logging_rank(), tid, __FILE__, __LINE__, \ + __func__); \ + } \ + } while(0); + +#define DEBUG_OUT(dstr, ...) \ + do { \ + if(dspaces_debug_enabled()) { \ + ABT_unit_id tid = 0; \ + if(ABT_initialized()) \ + ABT_thread_self_id(&tid); \ + fprintf(stderr, "Rank: %i TID: %" PRIu64 " %s:%i (%s): " dstr, \ + dspaces_logging_rank(), tid, __FILE__, __LINE__, __func__, \ + ##__VA_ARGS__); \ + } \ + } while(0); + +int dspaecs_trace_enabled(); +int dspaces_debug_enabled(); +int dspaces_logging_rank(); +int dspaces_init_logging(int rank); + +#endif // _DSPACES_LOGGING_H diff --git a/include/dspaces-modules.h b/include/dspaces-modules.h new file mode 100644 index 00000000..c60a2d7b --- /dev/null +++ b/include/dspaces-modules.h @@ -0,0 +1,92 @@ +#ifndef __DSPACES_MODULES_H__ +#define __DSPACES_MODULES_H__ + +#ifdef DSPACES_HAVE_PYTHON +#include "Python.h" +#endif // DSPACES_HAVE_PYTHON +#include "list.h" +#include "ss_data.h" + +typedef enum dspaces_mod_type { DSPACES_MOD_PY } dspaces_mod_type_t; + +struct dspaces_module { + struct list_head entry; + char *name; + char *namespace; + char *file; + dspaces_mod_type_t type; + union { +#ifdef DSPACES_HAVE_PYTHON + PyObject *pModule; +#endif // DSPACES_HAVE_PYTHON + }; +}; + +typedef enum dspaces_module_arg_type { + DSPACES_ARG_REAL, + DSPACES_ARG_INT, + DSPACES_ARG_STR, + DSPACES_ARG_NONE +} dspaces_mod_arg_type_t; + +struct dspaces_module_args { + char *name; + dspaces_mod_arg_type_t type; + int len; + union { + double rval; + long ival; + double *rarray; + long *iarray; + char *strval; + }; +}; + +typedef enum dspaces_module_return_type { + DSPACES_MOD_RET_ARRAY, + DSPACES_MOD_RET_INT, + DSPACES_MOD_RET_NONE, + DSPACES_MOD_RET_ERR +} dspaces_mod_ret_type_t; + +struct dspaces_module_ret { + dspaces_mod_ret_type_t type; + int len; + uint64_t *dim; + union { + int ndim; + int err; + }; + int tag; + int elem_size; + enum storage_type st; + union { + void *data; + long ival; + }; +}; + +int dspaces_init_mods(struct list_head *mods); + +int build_module_args_from_odsc(obj_descriptor *odsc, + struct dspaces_module_args **argsp); + +int build_module_arg_from_rank(long rank, struct dspaces_module_args *arg); + +int build_module_args_from_reg(reg_in_t *reg, + struct dspaces_module_args **argsp); + +void free_arg_list(struct dspaces_module_args *args, int len); + +struct dspaces_module *dspaces_mod_by_od(struct list_head *mods, + obj_descriptor *odsc); + +struct dspaces_module *dspaces_mod_by_name(struct list_head *mods, + const char *name); + +struct dspaces_module_ret *dspaces_module_exec(struct dspaces_module *mod, + const char *operation, + struct dspaces_module_args *args, + int nargs, int ret_type); + +#endif // __DSPACES_MODULES_H__ diff --git a/include/dspaces-remote.h b/include/dspaces-remote.h new file mode 100644 index 00000000..dc73b484 --- /dev/null +++ b/include/dspaces-remote.h @@ -0,0 +1,12 @@ +#ifndef __DSPACES_REMOTE_H +#define __DSPACES_REMOTE_H + +#include "dspaces.h" + +struct remote { + char *name; + char addr_str[128]; + dspaces_client_t conn; +}; + +#endif // __DSPACES_REMOTE_H \ No newline at end of file diff --git a/include/dspaces.h b/include/dspaces.h index 7de9092e..f8484420 100644 --- a/include/dspaces.h +++ b/include/dspaces.h @@ -308,13 +308,21 @@ struct dspaces_req { int ndim; uint64_t *lb, *ub; void *buf; + int tag; }; +int dspaces_get_req(dspaces_client_t client, struct dspaces_req *in_req, + struct dspaces_req *out_req, int timeout); + int dspaces_pexec(dspaces_client_t client, const char *var_name, unsigned int ver, int ndim, uint64_t *lb, uint64_t *ub, const char *fn, unsigned int fnsz, const char *fn_name, void **data, int *data_size); +int dspaces_mpexec(dspaces_client_t client, int num_args, + struct dspaces_req *args, const char *fn, unsigned int fnsz, + const char *fn_name, void **data, int *size); + typedef int (*dspaces_sub_fn)(dspaces_client_t, struct dspaces_req *, void *); typedef struct dspaces_sub_handle *dspaces_sub_t; #define DSPACES_SUB_FAIL NULL @@ -454,6 +462,9 @@ int dspaces_get_var_names(dspaces_client_t client, char ***var_names); int dspaces_get_var_objs(dspaces_client_t client, const char *name, struct dspaces_obj **objs); +long dspaces_register_simple(dspaces_client_t client, const char *type, + const char *name, const char *data, char **nspace); + #if defined(__cplusplus) } #endif diff --git a/include/ss_data.h b/include/ss_data.h index 74531116..4be37eae 100644 --- a/include/ss_data.h +++ b/include/ss_data.h @@ -46,6 +46,7 @@ typedef struct { enum storage_type st; uint32_t flags; int tag; + int type; char owner[128]; unsigned int version; @@ -199,7 +200,8 @@ struct dht { enum sspace_hash_version { ssd_hash_version_auto = 0, - ssd_hash_version_v1, // decompose the global data domain + ssd_hash_version_v0, // no hashing + ssd_hash_version_v1, // decompose the global data domain // using hilbert SFC ssd_hash_version_v2, // decompose the global data domain using // recursive bisection of the longest dimension @@ -420,6 +422,9 @@ MERCURY_GEN_PROC(pexec_out_t, ((hg_bulk_t)(handle))((int32_t)(length))( MERCURY_GEN_PROC(ss_information, ((odsc_hdr)(ss_buf))((hg_string_t)(chk_str))) MERCURY_GEN_PROC(cond_in_t, ((uint64_t)(mtxp))((uint64_t)(condp))) MERCURY_GEN_PROC(get_var_objs_in_t, ((hg_string_t)(var_name))((int32_t)(src))) +MERCURY_GEN_PROC(reg_in_t, + ((hg_string_t)(type))((hg_string_t)(name))( + (hg_string_t)(reg_data))((int32_t)(src))((uint64_t)(id))) char *obj_desc_sprint(obj_descriptor *); // diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index 19830d8e..114b5ccf 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -1,4 +1,4 @@ set(DSPACES_INSTALL_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/share/modules") -set(script-mods s3nc_mod.py) +set(script-mods s3nc_mod.py azure_mod.py url_mod.py ds_reg.py) install(FILES ${script-mods} DESTINATION ${DSPACES_INSTALL_MODULE_PATH}) diff --git a/modules/azure_mod.py b/modules/azure_mod.py new file mode 100644 index 00000000..ae60c726 --- /dev/null +++ b/modules/azure_mod.py @@ -0,0 +1,156 @@ +import planetary_computer +from netCDF4 import Dataset +import fsspec +import pystac_client +import numpy as np +import os +from urllib.parse import urlparse +from urllib.request import urlretrieve +from bitstring import Bits, pack +from datetime import date, timedelta +import sys + +base_date = date(1950, 1, 1) +present_date = date(2015, 1, 1) +last_date = date(2100, 12, 31) +cache_base='.azrcache' + +try: + catalog = pystac_client.Client.open( + "https://planetarycomputer.microsoft.com/api/stac/v1", + modifier=planetary_computer.sign_inplace, + ) + collection = catalog.get_collection("nasa-nex-gddp-cmip6") + variable_list=collection.summaries.get_list("cmip6:variable") + model_list=collection.summaries.get_list("cmip6:model")[:10] + scenario_list=collection.summaries.get_list("cmip6:scenario") + have_pc = True +except pystac_client.exceptions.APIError: + print("don't have planetary computer api access") + have_pc = False + +def _get_gddp_time_ranges(version): + bits = Bits(uint=version, length=32) + start_day, span_days = bits.unpack('uint:16, uint:16') + start_date = base_date + timedelta(days = start_day) + end_date = start_date + timedelta(days = span_days) + if start_date > last_date: + raise ValueError(f'WARNING: start_date of {start_date} is too far in the future') + if end_date > last_date: + print(f"WARNING: truncating end_date of {end_date} to {last_date}") + end_date = last_date + return start_date, end_date + +def _get_gddp_params(name): + model = 'CESM2' + scenario = 'ssp585' + variable = None + var_name = name.split('\\')[-1] + quality = 0 + name_parts = var_name.split(',') + for part in name_parts: + if part[0] == 'm': + model = part[2:] + if have_pc and model not in model_list: + raise ValueError(f"model {model} not available.") + if part[0] == 's': + scenario = part[2:] + if have_pc and scenario not in scenario_list: + raise ValueError(f"scenario {scenario} not available.") + if part[0] == 'v': + variable = part[2:] + if have_pc and variable not in variable_list: + raise ValueError(f"variable {variable} not available.") + if part[0] == 'q': + quality = int(part[2:]) + if variable == None: + raise ValueError('No variable name specified') + return model, scenario, variable, quality + +def _get_dataset(url): + path = urlparse(url).path + cache_entry = f'{cache_base}/{path}' + if not os.path.exists(cache_entry): + cache_dir = os.path.dirname(cache_entry) + if not os.path.exists(cache_dir): + os.makedirs(os.path.dirname(cache_entry)) + urlretrieve(url, filename=cache_entry) + return(Dataset(cache_entry)) + +def _get_azure_url(url, var_name): + ds = _get_dataset(url) + return(ds[var_name]) + +def _get_cmip6_data(model, scenario, variable, start_date, end_date, lb, ub): + result = None + result_days = (end_date - start_date).days + 1 + if have_pc: + search = catalog.search( + collections=["nasa-nex-gddp-cmip6"], + datetime=f'{start_date}/{end_date}', + query = { + "cmip6:model": { + "eq": model + }, + "cmip6:scenario": { + "in": ['historical', scenario] + }, + }, + sortby=[{'field':'cmip6:year','direction':'asc'}] + ) + items = search.item_collection() + + for item in items: + if have_pc: + year = item.properties['cmip6:year'] + url = item.assets[variable].href + ds = _get_dataset(url) + else: + # TODO - in case indexing is offline, we still want to hit the cache + pass + data = ds[variable] + if result is None: + if lb[0] >= data[0].shape[0] or lb[1] >= data[0].shape[1]: + return None + ub = (min(ub[0]+1, data[0].shape[0]), min(ub[1]+1, data[0].shape[1])) + shape = (result_days, ub[0] - lb[0], ub[1] - lb[1]) + result = np.ndarray(shape, dtype = data.dtype) + item_start = max(start_date, date(year, 1,1)) + item_end = min(date(year,12,31), end_date) + start_gidx = (item_start - start_date).days + end_gidx = (item_end - start_date).days + 1 + start_iidx = (item_start - date(year, 1 , 1)).days + end_iidx = (item_end - date(year, 1, 1)).days + 1 + result[start_gidx:end_gidx,:,:] = data[start_iidx:end_iidx,lb[0]:ub[0],lb[1]:ub[1]] + return(result) + +def validate(url, **kwargs): + pass + +def reg_query(name, version, lb, ub, params, url, var_name): + array = _get_azure_url(url, var_name) + if lb: + index = [ slice(lb[x], ub[x]+1) for x in range(len(lb)) ] + else: + index = [ slice(0, x) for x in array.shape] + return(array[index]) + + +def query(name, version, lb, ub): + start_date, end_date = _get_gddp_time_ranges(version) + model, scenario, variable, quality = _get_gddp_params(name) + result = _get_cmip6_data(model, scenario, variable, start_date, end_date, lb, ub) + return(result) + +if __name__ == '__main__': + s = date(2013, 5, 2) + e = date(2013, 5, 2) + start = (s - base_date).days + span = (e - s).days + lb = (0,0) + ub = (599,1399) + version = pack('uint:16, uint:16', start, span).uint + res = query(name='cmip6-planetary\\m:ACCESS-ESM1-5,v:tas', version=1, lb=lb, ub=ub) + print(res.shape) + + diff --git a/modules/ds_reg.py b/modules/ds_reg.py new file mode 100644 index 00000000..1db5c3a3 --- /dev/null +++ b/modules/ds_reg.py @@ -0,0 +1,90 @@ +import sys +import json +import importlib +import sqlite3 + +class Registry: + def __init__(self, db_name = 'dspaces_reg.db'): + self.db_name = db_name + db_conn = sqlite3.connect(db_name) + cursor = db_conn.cursor() + cursor.execute("CREATE TABLE IF NOT EXISTS type (id INTEGER PRIMARY KEY, name TEXT(256), module TEXT(256))") + cursor.execute("CREATE TABLE IF NOT EXISTS registry (id INTEGER PRIMARY KEY, reg_type INTEGER, name TEXT(256), data BLOCK, FOREIGN KEY(reg_type) REFERENCES type(id), UNIQUE(reg_type, name))") + self.modules = {} + for (name, module) in cursor.execute("SELECT name, module FROM type"): + self.modules[name] = importlib.import_module(module) + db_conn.commit() + cursor.close() + + def Add(self, reg_id, reg_type, name, data): + db_conn = sqlite3.connect(self.db_name) + cursor = db_conn.cursor() + if reg_type not in self.modules: + cursor.execute("INSERT INTO type(name, module) VALUES(?, ?)", (reg_type, reg_type)) + type_id = cursor.lastrowid + self.modules[reg_type] = importlib.import_module(reg_type) + else: + res = cursor.execute("SELECT id FROM type WHERE name == ?", (reg_type,)) + type_id, = res.fetchone() + self.modules[reg_type].validate(**json.loads(data)) + res = cursor.execute("SELECT id FROM registry WHERE reg_type == ? AND name == ?", (type_id, name)).fetchone() + if res is None: + cursor.execute("INSERT INTO registry(id, reg_type, name, data) VALUES(?, ?, ?, ?) ON CONFLICT DO NOTHING", (reg_id, type_id, name, data)) + else: + reg_id, = res + db_conn.commit() + cursor.close() + return(reg_id) + + def Query(self, reg_id, version, lb, ub, params): + db_conn = sqlite3.connect(self.db_name) + cursor = db_conn.cursor() + res = cursor.execute("SELECT type.name, registry.name, registry.data FROM registry INNER JOIN type ON registry.reg_type == type.id WHERE registry.id == ?", (reg_id,)) + (reg_type, name, ser_data) = res.fetchone() + db_conn.close() + data = json.loads(ser_data) + module = self.modules[reg_type] + return(module.reg_query(name, version, lb, ub, params, **data)) + + def HighestIDBetween(self, min_id, max_id): + db_conn = sqlite3.connect(self.db_name) + cursor = db_conn.cursor() + res = cursor.execute("SELECT MAX(id) FROM registry WHERE id >= ? AND id <= ?", (min_id, max_id)) + reg_id, = res.fetchone() + if reg_id is None: + reg_id = 0 + return(reg_id) + +reg = Registry() + +def register(type, name, data, id): + return(reg.Add(id, type, name, data)) + +def _get_query_params(name): + param_part = name.split('\\')[-1] + parts = param_part.split(',') + params = {} + for part in parts: + key, val = part.split(':',1 ) + if key == 'id': + id = val + else: + params[key] = val + return int(id), params + +def bootstrap_id(rank): + min_id = rank << 40 + max_id = ((rank + 1) << 40) - 1 + reg_id = reg.HighestIDBetween(min_id, max_id) + return(reg_id + 1) + +def query(name, version, lb, ub): + id_part = name.split('\\')[-1] + id, params = _get_query_params(name) + return(reg.Query(id, version, lb, ub, params)) + +if __name__ == '__main__': + register('s3nc_mod', 'abc', json.dumps({'bucket':'noaa-goes17','path':'ABI-L1b-RadM/2020/215/15/OR_ABI-L1b-RadM1-M6C02_G17_s20202151508255_e20202151508312_c20202151508338.nc'}), 45) + res = query('abcdef\id:45,var_name:Rad', 2, (1,1), (4,2)) + print(res) + diff --git a/modules/s3nc_mod.py b/modules/s3nc_mod.py index ce3cea49..c731523c 100644 --- a/modules/s3nc_mod.py +++ b/modules/s3nc_mod.py @@ -84,8 +84,26 @@ def build_cache_entry(dir_base, file_base, fcount): cache_file = f'{file_base}_G17_{fcount}.nc' return(f'{cache_dir}/{cache_file}') +def validate(bucket, path): + if not path.endswith('.nc'): + raise ValueError("S3 module can only access netCDF files.") + +def reg_query(name, version, lb, ub, params, bucket, path): + if 'var_name' not in params: + raise ValueError + var_name = params['var_name'] + cache_entry = f'{cache_base}/{bucket}/{path}' + if not os.path.exists(cache_entry): + s3.get(f'{bucket}/{path}', cache_entry) + data = Dataset(cache_entry) + array = data[var_name] + if lb: + index = [ slice(lb[x], ub[x]+1) for x in range(len(lb)) ] + else: + index = [ slice(0, x) for x in array.shape] + return(array[index]) + def query(name, version, lb, ub): - sys.stdout.flush() dir_base, file_base = build_dir_file(name, version) times = unpack_version(version) fcount = max(times[-2], times[-1]) @@ -97,18 +115,13 @@ def query(name, version, lb, ub): var = name.split('/')[-1] data = Dataset(centry) array = data[var] - print(lb, ub) if lb != None: index = [ slice(lb[x], ub[x]+1) for x in range(len(lb)) ] else: index = [ slice(0, x) for x in array.shape ] - print(f'index = {index}') - print(f'result shape = {array[index].shape}') - sys.stdout.flush() - sys.stderr.flush() return(array[index]) if __name__ == '__main__': - var_name = 's3nc\\RadM/M1/C2/Rad' + var_name = 'goes17\\RadM/M1/C2/Rad' version = 505081608 print(query(var_name, version, (1,1), (4,2))) diff --git a/modules/url_mod.py b/modules/url_mod.py new file mode 100644 index 00000000..0fee9595 --- /dev/null +++ b/modules/url_mod.py @@ -0,0 +1,54 @@ +from netCDF4 import Dataset +from urllib.parse import urlparse +from urllib.request import urlretrieve +import os + +cache_base = '.url' + +def _get_url_params(name): + param_ps_list = name.split('\\')[-1] + params_raw = param_ps_list.split('|') + url = None + var_name = None + for p in params_raw: + if p[0] == 'u': + url = p[2:] + if p[0] == 'v': + var_name = p[2:] + if not url: + raise ValueError('no url found') + return(url, var_name) + +def _get_url_array(url, name, var_name): + url_obj = urlparse(url) + cache_entry = f'{cache_base}/{name}/{url_obj.path}' + if not os.path.exists(cache_entry): + cache_dir = os.path.dirname(cache_entry) + os.makedirs(cache_dir, exist_ok=True) + urlretrieve(url, cache_entry) + data = Dataset(cache_entry) + return(data[var_name]) + +def reg_query(name, version, lb, ub, params, url): + var_name = params['var_name'] + array = _get_url_array(url, name, var_name) + if lb: + index = [ slice(lb[x], ub[x]+1) for x in range(len(lb)) ] + else: + index = [ slice(0, x) for x in array.shape] + return(array[index]) + +def validate(url, **kwargs): + url_obj = urlparse(url) + if not url_obj.path.endswith('.nc'): + raise ValueError("URL module can only access netCDF files.") + +def query(name, version, lb, ub): + url, var_name = _get_url_params(name) + validate(url) + return(reg_query(None, version, lb, ub, {'var_name': var_name}, url)) + +if __name__ == '__main__': + url = 'https://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc' + a = reg_query('foo', 0, (10, 10), (20,30), {'var_name':'area'}, url) + print(a) \ No newline at end of file diff --git a/scripts/build-and-run.sh b/scripts/build-and-run.sh index d24c8b2c..c547e00c 100755 --- a/scripts/build-and-run.sh +++ b/scripts/build-and-run.sh @@ -18,8 +18,12 @@ make install fi_info cd / -${install_dir}/bin/dspaces_server ${ofi_str} & -while [ -f conf.ds ] ; do +if [ -f dspaces.toml ] ; then + ${install_dir}/bin/dspaces_server ${ofi_str} dspaces.toml & +else + ${install_dir}/bin/dspaces_server ${ofi_str} & +fi +while [ ! -f conf.ds ] ; do sleep 1 done sleep 2 diff --git a/scripts/format.sh b/scripts/format.sh index b3db4dc9..9f48c1e1 100755 --- a/scripts/format.sh +++ b/scripts/format.sh @@ -5,6 +5,7 @@ if [ "$1" == "cf" ] ; then find src tests bindings -not -iname '*toml.c' -a \( -iname '*.h' -o -iname '*.c' \) | xargs clang-format -i if [ -n "$(git diff)" ] ; then echo "clang-format check failed" + echo "$(git diff)" exit -1 fi fi diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 09266e8d..2d27e705 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -75,7 +75,7 @@ set_target_properties (dspaces PROPERTIES VERSION ${dspaces_VERSION} SOVERSION ${dspaces_VERSION_MAJOR}) -set(dspaces-server-src util.c bbox.c ss_data.c dspaces-server.c toml.c) +set(dspaces-server-src util.c bbox.c ss_data.c dspaces-server.c dspaces-conf.c dspaces-modules.c dspaces-logging.c toml.c) add_library(dspaces-server ${dspaces-server-src} ${dspaces-src}) @@ -84,17 +84,27 @@ target_include_directories (dspaces-server BEFORE PUBLIC $ ${CURL_INCLUDE_DIR}) if(HAVE_DRC) - target_link_libraries (dspaces-server ${DRC_LIBRARIES}) - target_include_directories (dspaces-server BEFORE PUBLIC ${DRC_INCLUDE_DIRS}) + target_link_libraries (dspaces-server margo m pthread ${DRC_LIBRARIES} liblz4) + target_include_directories (dspaces-server BEFORE PUBLIC + $ ${DRC_INCLUDE_DIRS}) +else() + target_link_libraries (dspaces-server margo m pthread liblz4) + target_include_directories (dspaces-server BEFORE PUBLIC + $) endif() + if(Python_FOUND) - message(" ${PYTHON_INCLUDE_PATH} ${_Python_NumPy_INCLUDE_DIR}") target_link_libraries(dspaces-server Python::Python) target_include_directories (dspaces-server BEFORE PUBLIC ${PYTHON_INCLUDE_PATH} ${_Python_NumPy_INCLUDE_DIR}) target_compile_definitions(dspaces-server PRIVATE DSPACES_HAVE_PYTHON) endif() +if(CURL_FOUND) + target_link_libraries(dspaces-server curl) + target_include_directories (dspaces-server BEFORE PUBLIC ${CURL_INCLUDE_DIR}) +endif() + target_compile_definitions(dspaces-server PRIVATE DSPACES_MOD_DIR=${CMAKE_INSTALL_PREFIX}/share/modules) # for shared libs, establish the lib version diff --git a/src/bbox.c b/src/bbox.c index b00f0d25..c72292d8 100644 --- a/src/bbox.c +++ b/src/bbox.c @@ -41,7 +41,8 @@ #include "sfc.h" // static inline unsigned int -static inline uint64_t coord_dist(struct coord *c0, struct coord *c1, int dim) +static inline uint64_t coord_dist(const struct coord *c0, + const struct coord *c1, int dim) { return (c1->c[dim] - c0->c[dim] + 1); } @@ -181,7 +182,7 @@ int bbox_equals(const struct bbox *bb0, const struct bbox *bb1) return 0; } -uint64_t bbox_volume(struct bbox *bb) +uint64_t bbox_volume(const struct bbox *bb) { uint64_t n = 1; int ndims = bb->num_dims; diff --git a/src/dspaces-client.c b/src/dspaces-client.c index 3b7b5137..faad29b7 100644 --- a/src/dspaces-client.c +++ b/src/dspaces-client.c @@ -218,9 +218,11 @@ struct dspaces_client { hg_id_t notify_id; hg_id_t do_ops_id; hg_id_t pexec_id; + hg_id_t mpexec_id; hg_id_t cond_id; hg_id_t get_vars_id; hg_id_t get_var_objs_id; + hg_id_t reg_id; struct dc_gspace *dcg; char **server_address; char **node_names; @@ -377,7 +379,9 @@ static int init_ss_info(dspaces_client_t client) int ret; ret = get_ss_info(client, &ss_data); - install_ss_info(client, &ss_data); + if(ret == dspaces_SUCCESS) { + install_ss_info(client, &ss_data); + } return (ret); } @@ -396,6 +400,7 @@ static int init_ss_info_mpi(dspaces_client_t client, MPI_Comm comm) MPI_Bcast(&ss_data, sizeof(ss_data), MPI_BYTE, 0, comm); install_ss_info(client, &ss_data); } + return (ret); } @@ -818,11 +823,14 @@ static int dspaces_init_margo(dspaces_client_t client, &flag); margo_registered_name(client->mid, "pexec_rpc", &client->pexec_id, &flag); + margo_registered_name(client->mid, "mpexec_rpc", &client->pexec_id, + &flag); margo_registered_name(client->mid, "cond_rpc", &client->cond_id, &flag); margo_registered_name(client->mid, "get_vars_rpc", &client->get_vars_id, &flag); margo_registered_name(client->mid, "get_var_objs_rpc", &client->get_var_objs_id, &flag); + margo_registered_name(client->mid, "reg_rpc", &client->reg_id, &flag); } else { client->put_id = MARGO_REGISTER(client->mid, "put_rpc", bulk_gdim_t, bulk_out_t, NULL); @@ -874,6 +882,8 @@ static int dspaces_init_margo(dspaces_client_t client, do_ops_in_t, bulk_out_t, NULL); client->pexec_id = MARGO_REGISTER(client->mid, "pexec_rpc", pexec_in_t, pexec_out_t, NULL); + client->mpexec_id = MARGO_REGISTER(client->mid, "mpexec_rpc", + pexec_in_t, pexec_out_t, NULL); client->cond_id = MARGO_REGISTER(client->mid, "cond_rpc", cond_in_t, void, NULL); margo_registered_disable_response(client->mid, client->cond_id, @@ -882,6 +892,8 @@ static int dspaces_init_margo(dspaces_client_t client, int32_t, name_list_t, NULL); client->get_var_objs_id = MARGO_REGISTER( client->mid, "get_var_objs_rpc", get_var_objs_in_t, odsc_hdr, NULL); + client->reg_id = + MARGO_REGISTER(client->mid, "reg_rpc", reg_in_t, uint64_t, NULL); } return (dspaces_SUCCESS); @@ -943,7 +955,10 @@ int dspaces_init(int rank, dspaces_client_t *c) } choose_server(client); - init_ss_info(client); + ret = init_ss_info(client); + if(ret != dspaces_SUCCESS) { + return (ret); + } dspaces_post_init(client); *c = client; @@ -984,7 +999,10 @@ int dspaces_init_mpi(MPI_Comm comm, dspaces_client_t *c) } choose_server(client); - init_ss_info_mpi(client, comm); + ret = init_ss_info_mpi(client, comm); + if(ret != dspaces_SUCCESS) { + return (ret); + } dspaces_post_init(client); *c = client; @@ -1032,7 +1050,10 @@ int dspaces_init_wan(const char *listen_addr_str, const char *conn_str, } choose_server(client); - init_ss_info(client); + ret =init_ss_info(client); + if(ret != dspaces_SUCCESS) { + return (ret); + } dspaces_post_init(client); *c = client; @@ -1072,7 +1093,10 @@ int dspaces_init_wan_mpi(const char *listen_addr_str, const char *conn_str, } choose_server(client); - init_ss_info_mpi(client, comm); + ret = init_ss_info_mpi(client, comm); + if(ret != dspaces_SUCCESS) { + return (ret); + } dspaces_post_init(client); *c = client; @@ -1199,7 +1223,7 @@ static void copy_var_name_to_odsc(dspaces_client_t client, const char *var_name, obj_descriptor *odsc) { if(client->nspace) { - strncpy(odsc->name, client->nspace, strlen(client->nspace)); + strncpy(odsc->name, client->nspace, sizeof(odsc->name) - 1); odsc->name[strlen(client->nspace)] = '\\'; strncpy(&odsc->name[strlen(client->nspace) + 1], var_name, (sizeof(odsc->name) - strlen(client->nspace)) - 1); @@ -1274,13 +1298,20 @@ static int dspaces_cpu_put_tag(dspaces_client_t client, const char *var_name, hg_return_t hret; bulk_gdim_t in; bulk_out_t out; + int type; int ret = dspaces_SUCCESS; + if(elem_size < 0) { + type = elem_size; + elem_size = type_to_size(type); + } + obj_descriptor odsc = {.version = ver, .owner = {0}, .st = st, .flags = 0, .tag = tag, + .type = type, .size = elem_size, .bb = { .num_dims = ndim, @@ -1884,11 +1915,11 @@ static int get_data(dspaces_client_t client, int num_odscs, rdma_size[i] = (req_obj.size) * bbox_volume(&odsc_tab[i].bb); - DEBUG_OUT("For odsc %i, element size is %zi, and there are %li " - "elements to fetch.\n", + DEBUG_OUT("For odsc %i, element size is %zi, and there are %" PRIu64 + " elements to fetch.\n", i, req_obj.size, bbox_volume(&odsc_tab[i].bb)); - DEBUG_OUT("creating bulk handle for buffer %p of size %li.\n", + DEBUG_OUT("creating bulk handle for buffer %p of size %" PRIu64 ".\n", od[i]->data, rdma_size[i]); hret = margo_bulk_create(client->mid, 1, (void **)(&(od[i]->data)), @@ -1934,7 +1965,8 @@ static int get_data(dspaces_client_t client, int num_odscs, // decompress into buffer and copy back ret = LZ4_decompress_safe(od[i]->data, ucbuffer, resp.len, rdma_size[i]); - DEBUG_OUT("decompressed from %li to %i bytes\n", resp.len, ret); + DEBUG_OUT("decompressed from %" PRIu64 " to %i bytes\n", resp.len, + ret); if(ret != rdma_size[i]) { fprintf(stderr, "LZ4 decompression failed with %i.\n", ret); } @@ -2582,7 +2614,7 @@ int dspaces_put_local(dspaces_client_t client, const char *var_name, }}; hg_addr_t owner_addr; - size_t owner_addr_size = 128; + hg_size_t owner_addr_size = 128; margo_addr_self(client->mid, &owner_addr); margo_addr_to_string(client->mid, odsc.owner, &owner_addr_size, owner_addr); @@ -2663,8 +2695,8 @@ static int get_odscs(dspaces_client_t client, obj_descriptor *odsc, int timeout, in.odsc_gdim.size = sizeof(*odsc); in.odsc_gdim.raw_odsc = (char *)odsc; in.param = timeout; - - DEBUG_OUT("starting query.\n"); + + DEBUG_OUT("starting query.\n"); set_global_dimension(&(client->dcg->gdim_list), odsc->name, &(client->dcg->default_gdim), &od_gdim); in.odsc_gdim.gdim_size = sizeof(od_gdim); @@ -2672,7 +2704,7 @@ static int get_odscs(dspaces_client_t client, obj_descriptor *odsc, int timeout, DEBUG_OUT("Found gdims.\n"); get_server_address(client, &server_addr); - + hret = margo_create(client->mid, server_addr, client->query_id, &handle); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: %s: margo_create() failed with %d.\n", __func__, @@ -2725,6 +2757,103 @@ static void fill_odsc(dspaces_client_t client, const char *var_name, copy_var_name_to_odsc(client, var_name, odsc); } +static void odsc_from_req(dspaces_client_t client, struct dspaces_req *req, + obj_descriptor *odsc) +{ + fill_odsc(client, req->var_name, req->ver, req->elem_size, req->ndim, + req->lb, req->ub, odsc); +} + +static void fill_req(dspaces_client_t client, obj_descriptor *odsc, void *data, + struct dspaces_req *req) +{ + int i; + + if(client->nspace) { + req->var_name = strdup(&odsc->name[strlen(client->nspace) + 2]); + } else { + req->var_name = strdup(odsc->name); + } + req->ver = odsc->version; + req->elem_size = odsc->size; + req->ndim = odsc->bb.num_dims; + req->lb = malloc(sizeof(*req->lb) * req->ndim); + req->ub = malloc(sizeof(*req->ub) * req->ndim); + for(i = 0; i < req->ndim; i++) { + req->lb[i] = odsc->bb.lb.c[i]; + req->ub[i] = odsc->bb.ub.c[i]; + } + req->buf = data; + req->tag = odsc->tag; +} + +int dspaces_get_req(dspaces_client_t client, struct dspaces_req *in_req, + struct dspaces_req *out_req, int timeout) +{ + obj_descriptor odsc = {0}; + obj_descriptor *odsc_tab; + int num_odscs; + int elem_size; + int num_elem = 1; + int i, j; + int ret = dspaces_SUCCESS; + void *data; + + odsc_from_req(client, in_req, &odsc); + + DEBUG_OUT("Querying %s with timeout %d\n", obj_desc_sprint(&odsc), timeout); + + num_odscs = get_odscs(client, &odsc, timeout, &odsc_tab); + + DEBUG_OUT("Finished query - need to fetch %d objects\n", num_odscs); + for(int i = 0; i < num_odscs; i++) { + if(odsc_tab[i].flags & DS_OBJ_RESIZE) { + DEBUG_OUT("the result is cropped.\n"); + memcpy(&odsc.bb, &odsc_tab[i].bb, sizeof(odsc_tab[i].bb)); + } + DEBUG_OUT("%s\n", obj_desc_sprint(&odsc_tab[i])); + } + + // send request to get the obj_desc + if(num_odscs != 0) { + elem_size = odsc_tab[0].size; + } else { + DEBUG_OUT("not setting element size because there are no result " + "descriptors.\n"); + data = NULL; + return (ret); + } + + odsc.size = elem_size; + DEBUG_OUT("element size is %zi\n", odsc.size); + for(i = 0; i < odsc.bb.num_dims; i++) { + num_elem *= (odsc.bb.ub.c[i] - odsc.bb.lb.c[i]) + 1; + } + DEBUG_OUT("data buffer size is %d\n", num_elem * elem_size); + + odsc.tag = odsc_tab[0].tag; + for(i = 1; i < num_odscs; i++) { + if(odsc_tab[i].tag != odsc_tab[0].tag) { + fprintf(stderr, "WARNING: multiple distinct tag values returned in " + "query result. Returning first one.\n"); + break; + } + } + + if(in_req->buf == NULL) { + data = malloc(num_elem * elem_size); + } else { + data = in_req->buf; + } + + get_data(client, num_odscs, odsc, odsc_tab, data); + if(out_req) { + fill_req(client, &odsc, data, out_req); + } + + return ret; +} + int dspaces_aget(dspaces_client_t client, const char *var_name, unsigned int ver, int ndim, uint64_t *lb, uint64_t *ub, void **data, int *tag, int timeout) @@ -2745,10 +2874,11 @@ int dspaces_aget(dspaces_client_t client, const char *var_name, DEBUG_OUT("Finished query - need to fetch %d objects\n", num_odscs); for(int i = 0; i < num_odscs; i++) { - if(odsc_tab[i].flags && DS_OBJ_RESIZE) { + if(odsc_tab[i].flags & DS_OBJ_RESIZE) { DEBUG_OUT("the result is cropped.\n"); memcpy(&odsc.bb, &odsc_tab[i].bb, sizeof(odsc_tab[i].bb)); for(j = 0; j < odsc.bb.num_dims; j++) { + lb[j] = odsc.bb.lb.c[j]; ub[j] = odsc.bb.ub.c[j]; } } @@ -2959,7 +3089,7 @@ int dspaces_get_meta(dspaces_client_t client, const char *name, int mode, DEBUG_OUT("Replied with version %d.\n", out.version); if(out.mdata.len) { - DEBUG_OUT("fetching %zi bytes.\n", out.mdata.len); + DEBUG_OUT("fetching %" PRIu64 " bytes.\n", out.mdata.len); *data = malloc(out.mdata.len); /* hret = margo_bulk_create(client->mid, 1, data, &out.size, @@ -3008,6 +3138,128 @@ int dspaces_get_meta(dspaces_client_t client, const char *name, int mode, return dspaces_ERR_MERCURY; } +int dspaces_mpexec(dspaces_client_t client, int num_args, + struct dspaces_req *args, const char *fn, unsigned int fnsz, + const char *fn_name, void **data, int *size) +{ + obj_descriptor *arg_odscs; + hg_addr_t server_addr; + hg_return_t hret; + hg_handle_t handle, cond_handle; + pexec_in_t in; + pexec_out_t out; + cond_in_t in2; + uint64_t mtxp, condp; + hg_bulk_t bulk_handle; + hg_size_t rdma_size; + margo_request req; + int i; + + in.odsc.size = num_args * sizeof(*arg_odscs); + in.odsc.raw_odsc = calloc(1, in.odsc.size); + arg_odscs = (obj_descriptor *)in.odsc.raw_odsc; + for(i = 0; i < num_args; i++) { + odsc_from_req(client, &args[i], &arg_odscs[i]); + DEBUG_OUT("Remote args %i is %s\n", i, obj_desc_sprint(&arg_odscs[i])); + } + + in.fn_name = strdup(fn_name); + + rdma_size = fnsz; + in.length = fnsz; + if(rdma_size > 0) { + DEBUG_OUT("sending fn\n"); + hret = margo_bulk_create(client->mid, 1, (void **)&fn, &rdma_size, + HG_BULK_READ_ONLY, &in.handle); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_bulk_create() failed\n", + __func__); + return dspaces_ERR_MERCURY; + } + DEBUG_OUT("created fn tranfer buffer\n"); + } + + get_server_address(client, &server_addr); + + hret = margo_create(client->mid, server_addr, client->mpexec_id, &handle); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); + margo_bulk_free(in.handle); + return dspaces_ERR_MERCURY; + } + + hret = margo_forward(handle, &in); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_forward() failed\n", __func__); + margo_bulk_free(in.handle); + margo_destroy(handle); + return dspaces_ERR_MERCURY; + } + + hret = margo_get_output(handle, &out); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_get_output() failed\n", __func__); + margo_bulk_free(in.handle); + margo_destroy(handle); + return dspaces_ERR_MERCURY; + } + margo_bulk_free(in.handle); + + DEBUG_OUT("received result with size %" PRIu32 "\n", out.length); + rdma_size = out.length; + if(rdma_size > 0) { + *data = malloc(out.length); + hret = margo_bulk_create(client->mid, 1, (void **)data, &rdma_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + if(hret != HG_SUCCESS) { + // TODO notify server of failure (server is waiting) + margo_free_output(handle, &out); + margo_destroy(handle); + return dspaces_ERR_MERCURY; + } + hret = margo_bulk_transfer(client->mid, HG_BULK_PULL, server_addr, + out.handle, 0, bulk_handle, 0, rdma_size); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_bulk_transfer failed!\n", + __func__); + margo_free_output(handle, &out); + margo_destroy(handle); + return dspaces_ERR_MERCURY; + } + margo_bulk_free(bulk_handle); + in2.mtxp = out.mtxp; + in2.condp = out.condp; + hret = margo_create(client->mid, server_addr, client->cond_id, + &cond_handle); + + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); + return dspaces_ERR_MERCURY; + } + + DEBUG_OUT("sending cond_rpc with condp = %" PRIu64 ", mtxp = %" PRIu64 + "\n", + in2.condp, in2.mtxp); + hret = margo_iforward(cond_handle, &in2, &req); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_iforward() failed\n", __func__); + margo_destroy(cond_handle); + return dspaces_ERR_MERCURY; + } + DEBUG_OUT("sent\n"); + *size = rdma_size; + margo_destroy(cond_handle); + } else { + *size = 0; + *data = NULL; + } + margo_free_output(handle, &out); + margo_destroy(handle); + free(in.odsc.raw_odsc); + DEBUG_OUT("done with handling pexec\n"); + return (dspaces_SUCCESS); +} + int dspaces_pexec(dspaces_client_t client, const char *var_name, unsigned int ver, int ndim, uint64_t *lb, uint64_t *ub, const char *fn, unsigned int fnsz, const char *fn_name, @@ -3129,6 +3381,59 @@ int dspaces_pexec(dspaces_client_t client, const char *var_name, return (dspaces_SUCCESS); } +long dspaces_register_simple(dspaces_client_t client, const char *type, + const char *name, const char *reg_data, + char **nspace) +{ + hg_addr_t server_addr; + hg_return_t hret; + reg_in_t in; + uint64_t out; + hg_handle_t handle; + long reg_handle; + + in.type = strdup(type); + in.name = strdup(name); + in.reg_data = strdup(reg_data); + in.src = -1; + + get_server_address(client, &server_addr); + + hret = margo_create(client->mid, server_addr, client->reg_id, &handle); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); + return (DS_MOD_ECLIENT); + } + + hret = margo_forward(handle, &in); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_forward() failed\n", __func__); + margo_destroy(handle); + return (DS_MOD_ECLIENT); + } + + hret = margo_get_output(handle, &out); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_get_output() failed\n", __func__); + margo_destroy(handle); + return (DS_MOD_ECLIENT); + } + + reg_handle = out; + if(nspace) { + if(reg_handle < 0) { + *nspace = NULL; + } else { + *nspace = strdup("ds_reg"); + } + } + + margo_free_output(handle, &out); + margo_destroy(handle); + + return (reg_handle); +} + static void get_local_rpc(hg_handle_t handle) { hg_return_t hret; @@ -3192,7 +3497,7 @@ static void get_local_rpc(hg_handle_t handle) hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); void *buffer = (void *)od->data; - DEBUG_OUT("creating buffer of size %zi\n", size); + DEBUG_OUT("creating buffer of size %" PRIu64 "\n", size); APEX_NAME_TIMER_START(4, "get_local_bulk_create"); hret = margo_bulk_create(mid, 1, (void **)&buffer, &size, HG_BULK_READ_ONLY, &bulk_handle); @@ -3221,7 +3526,7 @@ static void get_local_rpc(hg_handle_t handle) int req_id = 0; size_t offset = 0; while(remaining) { - DEBUG_OUT("%li bytes left to transfer\n", remaining); + DEBUG_OUT("%" PRIu64 " bytes left to transfer\n", remaining); offset = size - remaining; xfer_size = (remaining > BULK_TRANSFER_MAX) ? BULK_TRANSFER_MAX : remaining; @@ -3547,7 +3852,7 @@ struct dspaces_sub_handle *dspaces_sub(dspaces_client_t client, struct dspaces_sub_handle *subh; odsc_gdim_t in; struct global_dimension od_gdim; - size_t owner_addr_size = 128; + hg_size_t owner_addr_size = 128; int ret; if(client->listener_init == 0) { @@ -3588,7 +3893,7 @@ struct dspaces_sub_handle *dspaces_sub(dspaces_client_t client, memset(subh->q_odsc.bb.ub.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); memcpy(subh->q_odsc.bb.lb.c, lb, sizeof(uint64_t) * ndim); memcpy(subh->q_odsc.bb.ub.c, ub, sizeof(uint64_t) * ndim); - strncpy(subh->q_odsc.name, var_name, strlen(var_name) + 1); + strncpy(subh->q_odsc.name, var_name, sizeof(subh->q_odsc.name) - 1); copy_var_name_to_odsc(client, var_name, &subh->q_odsc); // A hack to send our address to the server without using more space. This @@ -3736,6 +4041,8 @@ int dspaces_op_calc(dspaces_client_t client, struct ds_data_expr *expr, hg_return_t hret; int ret; + DEBUG_OUT("Sending expression type %i\n", expr->type); + in.expr = expr; cbuf = malloc(expr->size); @@ -3767,8 +4074,9 @@ int dspaces_op_calc(dspaces_client_t client, struct ds_data_expr *expr, if(out.len) { *buf = malloc(rdma_size); ret = LZ4_decompress_safe(cbuf, *buf, out.len, rdma_size); - DEBUG_OUT("decompressed results from %zi to %zi bytes.\n", out.len, - rdma_size); + DEBUG_OUT("decompressed results from %" PRIu64 " to %" PRIu64 + " bytes.\n", + out.len, rdma_size); free(cbuf); } else { *buf = cbuf; @@ -3776,6 +4084,8 @@ int dspaces_op_calc(dspaces_client_t client, struct ds_data_expr *expr, margo_free_output(handle, &out); margo_destroy(handle); + + return (dspaces_SUCCESS); } void dspaces_set_namespace(dspaces_client_t client, const char *nspace) @@ -3820,7 +4130,7 @@ int dspaces_get_var_names(dspaces_client_t client, char ***var_names) return (-1); } - DEBUG_OUT("Received %zi variables in reply\n", out.count); + DEBUG_OUT("Received %" PRIu64 " variables in reply\n", out.count); *var_names = malloc(sizeof(*var_names) * out.count); for(i = 0; i < out.count; i++) { diff --git a/src/dspaces-conf.c b/src/dspaces-conf.c new file mode 100644 index 00000000..0fc68ac5 --- /dev/null +++ b/src/dspaces-conf.c @@ -0,0 +1,395 @@ +#include "dspaces-conf.h" +#include "dspaces-modules.h" +#include "dspaces-storage.h" +#include "toml.h" +#include +#include +#include +#ifdef DSPACES_HAVE_CURL +#include +#endif + +#define xstr(s) str(s) +#define str(s) #s + +static void eat_spaces(char *line) +{ + char *t = line; + + while(t && *t) { + if(*t != ' ' && *t != '\t' && *t != '\n') + *line++ = *t; + t++; + } + if(line) + *line = '\0'; +} + +static int parse_line(int lineno, char *line, struct ds_conf *conf) +{ + struct { + const char *opt; + int *pval; + } options[] = {{"ndim", &conf->ndim}, + {"dims", (int *)&conf->dims}, + {"max_versions", &conf->max_versions}, + {"hash_version", &conf->hash_version}, + {"num_apps", &conf->num_apps}}; + + char *t; + int i, n; + + /* Comment line ? */ + if(line[0] == '#') + return 0; + + t = strstr(line, "="); + if(!t) { + eat_spaces(line); + if(strlen(line) == 0) + return 0; + else + return -EINVAL; + } + + t[0] = '\0'; + eat_spaces(line); + t++; + + n = sizeof(options) / sizeof(options[0]); + + for(i = 0; i < n; i++) { + if(strcmp(line, options[1].opt) == 0) { /**< when "dims" */ + // get coordinates + int idx = 0; + char *crd; + crd = strtok(t, ","); + while(crd != NULL) { + ((struct coord *)options[1].pval)->c[idx] = atoll(crd); + crd = strtok(NULL, ","); + idx++; + } + if(idx != *(int *)options[0].pval) { + fprintf(stderr, "ERROR: (%s): dimensionality mismatch.\n", + __func__); + fprintf(stderr, "ERROR: index=%d, ndims=%d\n", idx, + *(int *)options[0].pval); + return -EINVAL; + } + break; + } + if(strcmp(line, options[i].opt) == 0) { + eat_spaces(line); + *(int *)options[i].pval = atoi(t); + break; + } + } + + if(i == n) { + fprintf(stderr, "WARNING: (%s): unknown option '%s' at line %d.\n", + __func__, line, lineno); + } + return 0; +} + +int parse_conf(const char *fname, struct ds_conf *conf) +{ + FILE *fin; + char buff[1024]; + int lineno = 1, err; + + fin = fopen(fname, "rt"); + if(!fin) { + fprintf(stderr, "ERROR: could not open configuration file '%s'.\n", + fname); + return -errno; + } + + while(fgets(buff, sizeof(buff), fin) != NULL) { + err = parse_line(lineno++, buff, conf); + if(err < 0) { + fclose(fin); + return err; + } + } + + fclose(fin); + return 0; +} + +static int get_toml_int(toml_table_t *t, const char *key, int *i) +{ + toml_datum_t dat; + + if(toml_key_exists(t, key)) { + dat = toml_int_in(t, key); + *i = dat.u.i; + return (1); + } + + return (0); +} + +static void get_toml_double(toml_table_t *t, const char *key, double *d) +{ + toml_datum_t dat; + + if(toml_key_exists(t, key)) { + dat = toml_double_in(t, key); + *d = dat.u.d; + } +} + +static void get_toml_str(toml_table_t *t, const char *key, char **str) +{ + toml_datum_t dat; + + if(toml_key_exists(t, key)) { + dat = toml_string_in(t, key); + *str = strdup(dat.u.s); + } +} + +static int get_toml_arr_ui64(toml_table_t *t, const char *key, uint64_t *vals) +{ + toml_array_t *arr; + toml_datum_t dat; + int i; + + arr = toml_array_in(t, key); + if(!arr) { + return (0); + } + for(i = 0;; i++) { + dat = toml_int_at(arr, i); + if(!dat.ok) { + break; + } + vals[i] = dat.u.i; + } + return (i); +} + +static void parse_remotes_table(toml_table_t *remotes, struct ds_conf *conf) +{ + toml_table_t *remote; + char *ip; + int port; + int i; + + conf->nremote = toml_table_ntab(remotes); + *conf->remotes = malloc(sizeof(**conf->remotes) * conf->nremote); + for(i = 0; i < conf->nremote; i++) { + (*conf->remotes)[i].name = strdup(toml_key_in(remotes, i)); + remote = toml_table_in(remotes, (*conf->remotes)[i].name); + get_toml_str(remote, "ip", &ip); + get_toml_int(remote, "port", &port); + sprintf((*conf->remotes)[i].addr_str, "%s:%i", ip, port); + free(ip); + } +} + +static void parse_storage_table(toml_table_t *storage, struct ds_conf *conf) +{ + int ndir, nfile; + struct dspaces_dir *dir; + struct dspaces_file *file; + toml_table_t *conf_dir; + toml_array_t *arr; + toml_datum_t dat; + int i, j; + + ndir = toml_table_ntab(storage); + for(i = 0; i < ndir; i++) { + dir = malloc(sizeof(*dir)); + dir->name = strdup(toml_key_in(storage, i)); + conf_dir = toml_table_in(storage, dir->name); + get_toml_str(conf_dir, "directory", &dir->path); + if(0 != (arr = toml_array_in(conf_dir, "files"))) { + INIT_LIST_HEAD(&dir->files); + nfile = toml_array_nelem(arr); + for(j = 0; j < nfile; j++) { + file = malloc(sizeof(*file)); + file->type = DS_FILE_NC; + dat = toml_string_at(arr, j); + file->name = strdup(dat.u.s); + free(dat.u.s); + list_add(&file->entry, &dir->files); + } + } else { + dat = toml_string_in(conf_dir, "files"); + if(dat.ok) { + if(strcmp(dat.u.s, "all") == 0) { + dir->cont_type = DS_FILE_ALL; + } else { + fprintf(stderr, + "ERROR: %s: invalid value for " + "storage.%s.files: %s\n", + __func__, dir->name, dat.u.s); + } + free(dat.u.s); + } else { + fprintf(stderr, + "ERROR: %s: no readable 'files' key for '%s'.\n", + __func__, dir->name); + } + } + } +} + +static void parse_modules_table(toml_table_t *modules, struct ds_conf *conf) +{ + struct dspaces_module *mod; + toml_table_t *module; + char *server, *file, *url, *type, *ext; + char *fname; + char *arg_part; + FILE *mod_file; + int nmod; +#ifdef DSPACES_HAVE_CURL + CURL *curl = curl_easy_init(); +#endif + int i; + + nmod = toml_table_ntab(modules); + for(i = 0; i < nmod; i++) { + mod = calloc(1, sizeof(*mod)); + mod->name = strdup(toml_key_in(modules, i)); + module = toml_table_in(modules, mod->name); + get_toml_str(module, "namespace", &mod->namespace); + if(!mod->namespace) { + fprintf(stderr, + "WARNING: No namespace for '%s'. Query matching currently " + "requires a namespace.\n", + mod->name); + } + url = NULL; + file = NULL; + get_toml_str(module, "url", &url); + get_toml_str(module, "file", &file); + if(url) { +#ifdef DSPACES_HAVE_CURL + if(!file) { + file = strdup(strrchr(url, '/')) + 1; + arg_part = strchr(file, '?'); + if(arg_part) { + arg_part[0] = '\0'; + } + } + fname = malloc(strlen(xstr(DSPACES_MOD_DIR)) + strlen(file) + 2); + sprintf(fname, "%s/%s", xstr(DSPACES_MOD_DIR), file); + mod_file = fopen(fname, "wb"); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, mod_file); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_perform(curl); + fclose(mod_file); + free(fname); +#else + fprintf(stderr, + "WARNING: could not download module '%s': compiled without " + "curl support.\n", + mod->name); +#endif // DSPACES_HAVE_CURL + } + if(file) { + ext = strrchr(file, '.'); + if(strcmp(ext, ".py") == 0) { + mod->type = DSPACES_MOD_PY; + ext[0] = '\0'; + mod->file = file; + } else { + fprintf(stderr, + "WARNING: could not determine type of module '%s', " + "with extension '%s'. Skipping.\n", + mod->name, ext); + free(file); + continue; + } + } + list_add(&mod->entry, conf->mods); + } +#ifdef DSPACES_HAVE_CURL + curl_easy_cleanup(curl); +#endif +} + +int parse_conf_toml(const char *fname, struct ds_conf *conf) +{ + FILE *fin; + toml_table_t *toml_conf, *server, *remotes, *storage, *modules; + char errbuf[200]; + char *ip; + int port; + int ndir; + struct dspaces_dir *dir; + struct dspaces_file *file; + int i; + + fin = fopen(fname, "r"); + if(!fin) { + fprintf(stderr, "ERROR: could not open configuration file '%s'.\n", + fname); + return -errno; + } + + toml_conf = toml_parse_file(fin, errbuf, sizeof(errbuf)); + fclose(fin); + + if(!conf) { + fprintf(stderr, "could not parse %s, %s.\n", fname, errbuf); + return -1; + } + + server = toml_table_in(toml_conf, "server"); + if(!server) { + fprintf(stderr, "missing [server] block from %s\n", fname); + return -1; + } + + conf->ndim = get_toml_arr_ui64(server, "dims", conf->dims.c); + get_toml_int(server, "max_versions", &conf->max_versions); + get_toml_int(server, "hash_version", &conf->hash_version); + get_toml_int(server, "num_apps", &conf->num_apps); + + remotes = toml_table_in(toml_conf, "remotes"); + if(remotes) { + parse_remotes_table(remotes, conf); + } + + storage = toml_table_in(toml_conf, "storage"); + if(storage) { + parse_storage_table(storage, conf); + } + + modules = toml_table_in(toml_conf, "modules"); + if(modules) { + parse_modules_table(modules, conf); + } + + toml_free(toml_conf); + + return (0); +} + +void print_conf(struct ds_conf *conf) +{ + int i; + + printf("DataSpaces server config:\n"); + printf("=========================\n"); + printf(" Default global dimensions: ("); + printf("%" PRIu64, conf->dims.c[0]); + for(i = 1; i < conf->ndim; i++) { + printf(", %" PRIu64, conf->dims.c[i]); + } + printf(")\n"); + printf(" MAX STORED VERSIONS: %i\n", conf->max_versions); + printf(" HASH TYPE: %s\n", hash_strings[conf->hash_version]); + if(conf->num_apps >= 0) { + printf(" APPS EXPECTED: %i\n", conf->num_apps); + } else { + printf(" RUN UNTIL KILLED\n"); + } + printf("=========================\n"); + fflush(stdout); +} diff --git a/src/dspaces-config.cmake.in b/src/dspaces-config.cmake.in index e29e0545..5c07710e 100644 --- a/src/dspaces-config.cmake.in +++ b/src/dspaces-config.cmake.in @@ -8,5 +8,6 @@ set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}") include (CMakeFindDependencyMacro) include (xpkg-import) xpkg_import_module (margo REQUIRED margo) +xpkg_import_module (liblz4 REQUIRED liblz4) include ("${CMAKE_CURRENT_LIST_DIR}/dspaces-targets.cmake") diff --git a/src/dspaces-logging.c b/src/dspaces-logging.c new file mode 100644 index 00000000..302e8601 --- /dev/null +++ b/src/dspaces-logging.c @@ -0,0 +1,43 @@ +#include "dspaces-logging.h" + +#include +#include +#include +#include + +static atomic_int f_dsp_debug = 0; +static atomic_int f_dsp_trace = 0; +static atomic_int dsp_log_rank = -1; + +int dspaces_debug_enabled() { return (f_dsp_debug); } + +int dspaces_trace_enabled() { return (f_dsp_trace); } + +int dspaces_logging_rank() { return (dsp_log_rank); } + +int dspaces_init_logging(int rank) +{ + const char *envdebug = getenv("DSPACES_DEBUG"); + const char *envtrace = getenv("DSPACES_TRACE"); + int err; + + dsp_log_rank = rank; + + if(envdebug) { + f_dsp_debug = 1; + } else { + f_dsp_debug = 0; + } + + if(envtrace) { + f_dsp_trace = 1; + } else { + f_dsp_trace = 0; + } + + DEBUG_OUT("initialized logging.\n"); + + return (0); +err_out: + return (err); +} diff --git a/src/dspaces-modules.c b/src/dspaces-modules.c new file mode 100644 index 00000000..112742b7 --- /dev/null +++ b/src/dspaces-modules.c @@ -0,0 +1,474 @@ +#include "dspaces-modules.h" +#include "dspaces-common.h" +#include "dspaces-logging.h" +#include "ss_data.h" + +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION + +#ifdef DSPACES_HAVE_PYTHON +#include +#include +#include + +#define xstr(s) str(s) +#define str(s) #s + +static int dspaces_init_py_mod(struct dspaces_module *mod) +{ + PyObject *pName; + + if(!mod->file) { + return (-1); + } + + pName = PyUnicode_DecodeFSDefault(mod->file); + mod->pModule = PyImport_Import(pName); + if(!mod->pModule) { + fprintf( + stderr, + "WARNING: could not load module '%s' from %s. File missing? Any " + "%s accesses will fail.\n", + mod->file, xstr(DSPACES_MOD_DIR), mod->name); + PyErr_Print(); + return (-1); + } + Py_DECREF(pName); + + DEBUG_OUT("initialized module '%s'.\n", mod->name); + + return (0); +} +#endif // DSPACES_HAVE_PYTHON + +static void add_builtin_mods(struct list_head *mods) +{ + struct dspaces_module *builtins; + int nbuiltin = 0; + int i; +#ifdef DSPACES_HAVE_PYTHON + + nbuiltin++; +#endif // DSPACES_HAVE_PYTHON + builtins = calloc(nbuiltin, sizeof(*builtins)); + +#ifdef DSPACES_HAVE_PYTHON + builtins[nbuiltin - 1].file = strdup("ds_reg"); + builtins[nbuiltin - 1].name = strdup("ds_reg"); + builtins[nbuiltin - 1].namespace = strdup("ds_reg"); + builtins[nbuiltin - 1].type = DSPACES_MOD_PY; +#endif // DSPACES_HAVE_PYTHON + + for(i = 0; i < nbuiltin; i++) { + list_add(&builtins[i].entry, mods); + } +} + +int dspaces_init_mods(struct list_head *mods) +{ + struct dspaces_module *mod; + + add_builtin_mods(mods); + + list_for_each_entry(mod, mods, struct dspaces_module, entry) + { + switch(mod->type) { + case DSPACES_MOD_PY: +#ifdef DSPACES_HAVE_PYTHON + dspaces_init_py_mod(mod); +#else + fprintf(stderr, + "WARNING: Unable to load module '%s', which is a python " + "module. DataSpaces was compiled without Python support.\n", + mod->name); +#endif // DSPACES_HAVE_PYTHON + break; + default: + fprintf(stderr, + "WARNING: unknown type %i for module '%s'. Corruption?\n", + mod->type, mod->name); + } + // TODO: unlink module on init failure? + } + + return (0); +} + +int build_module_arg_from_rank(long rank, struct dspaces_module_args *arg) +{ + arg->name = strdup("rank"); + arg->type = DSPACES_ARG_INT; + arg->len = -1; + arg->ival = rank; + + return (1); +} + +int build_module_args_from_reg(reg_in_t *reg, + struct dspaces_module_args **argsp) +{ + struct dspaces_module_args *args; + int nargs = 4; + int i; + + args = malloc(sizeof(*args) * nargs); + + // reg->type + args[0].name = strdup("type"); + args[0].type = DSPACES_ARG_STR; + args[0].len = strlen(reg->type) + 1; + args[0].strval = strdup(reg->type); + + // reg->name + args[1].name = strdup("name"); + args[1].type = DSPACES_ARG_STR; + args[1].len = strlen(reg->name) + 1; + args[1].strval = strdup(reg->name); + + // reg->reg_data + args[2].name = strdup("data"); + args[2].type = DSPACES_ARG_STR; + args[2].len = strlen(reg->reg_data) + 1; + args[2].strval = strdup(reg->reg_data); + + // reg->id + args[3].name = strdup("id"); + args[3].type = DSPACES_ARG_INT; + args[3].len = -1; + args[3].ival = reg->id; + + *argsp = args; + return (nargs); +} + +int build_module_args_from_odsc(obj_descriptor *odsc, + struct dspaces_module_args **argsp) +{ + struct dspaces_module_args *args; + int nargs = 4; + int ndims; + int i; + + args = malloc(sizeof(*args) * nargs); + + // odsc->name + args[0].name = strdup("name"); + args[0].type = DSPACES_ARG_STR; + args[0].len = strlen(odsc->name) + 1; + args[0].strval = strdup(odsc->name); + + // odsc->version + args[1].name = strdup("version"); + args[1].type = DSPACES_ARG_INT; + args[1].len = -1; + args[1].ival = odsc->version; + + ndims = odsc->bb.num_dims; + // odsc->bb.lb + args[2].name = strdup("lb"); + if(ndims > 0) { + args[2].name = strdup("lb"); + args[2].type = DSPACES_ARG_INT; + args[2].len = ndims; + args[2].iarray = malloc(sizeof(*args[2].iarray) * ndims); + for(i = 0; i < ndims; i++) { + if(odsc->st == row_major) { + args[2].iarray[i] = odsc->bb.lb.c[i]; + } else { + args[2].iarray[(ndims - i) - 1] = odsc->bb.lb.c[i]; + } + } + } else { + args[2].type = DSPACES_ARG_NONE; + } + + // odsc->bb.ub + args[3].name = strdup("ub"); + if(ndims > 0) { + args[3].type = DSPACES_ARG_INT; + args[3].len = ndims; + args[3].iarray = malloc(sizeof(*args[3].iarray) * ndims); + for(i = 0; i < ndims; i++) { + if(odsc->st == row_major) { + args[3].iarray[i] = odsc->bb.ub.c[i]; + } else { + args[3].iarray[(ndims - i) - 1] = odsc->bb.ub.c[i]; + } + } + } else { + args[3].type = DSPACES_ARG_NONE; + } + + *argsp = args; + return (nargs); +} + +static void free_arg(struct dspaces_module_args *arg) +{ + if(arg) { + free(arg->name); + if(arg->len > 0 && arg->type != DSPACES_ARG_NONE) { + free(arg->strval); + } + } else { + fprintf(stderr, "WARNING: trying to free NULL argument in %s\n", + __func__); + } +} + +void free_arg_list(struct dspaces_module_args *args, int len) +{ + int i; + + if(args) { + for(i = 0; i < len; i++) { + free_arg(&args[i]); + } + } else if(len > 0) { + fprintf(stderr, "WARNING: trying to free NULL argument list in %s\n", + __func__); + } +} + +static struct dspaces_module *find_mod(struct list_head *mods, + const char *mod_name) +{ + struct dspaces_module *mod; + int i; + + list_for_each_entry(mod, mods, struct dspaces_module, entry) + { + if(strcmp(mod_name, mod->name) == 0) { + return (mod); + } + } + + return (NULL); +} + +struct dspaces_module *dspaces_mod_by_od(struct list_head *mods, + obj_descriptor *odsc) +{ + struct dspaces_module *mod; + + DEBUG_OUT("finding module for odsc with name %s.\n", odsc->name); + list_for_each_entry(mod, mods, struct dspaces_module, entry) + { + // TODO: query mods for match + if(mod->namespace && strstr(odsc->name, mod->namespace) == odsc->name) { + DEBUG_OUT("found %s\n", mod->name); + return (mod); + } + } + + return (NULL); +} + +struct dspaces_module *dspaces_mod_by_name(struct list_head *mods, + const char *name) +{ + struct dspaces_module *mod; + + list_for_each_entry(mod, mods, struct dspaces_module, entry) + { + if(strcmp(mod->name, name) == 0) { + return (mod); + } + } + return (NULL); +} + +static struct dspaces_module_ret *ds_module_ret_err(int err) +{ + struct dspaces_module_ret *ret = malloc(sizeof(*ret)); + + ret->type = DSPACES_MOD_RET_ERR; + ret->err = err; + + return (ret); +} + +#ifdef DSPACES_HAVE_PYTHON +PyObject *py_obj_from_arg(struct dspaces_module_args *arg) +{ + PyObject *pArg; + int i; + + switch(arg->type) { + case DSPACES_ARG_REAL: + if(arg->len == -1) { + return (PyFloat_FromDouble(arg->rval)); + } + pArg = PyTuple_New(arg->len); + for(i = 0; i < arg->len; i++) { + PyTuple_SetItem(pArg, i, PyFloat_FromDouble(arg->rarray[i])); + } + return (pArg); + case DSPACES_ARG_INT: + if(arg->len == -1) { + return (PyLong_FromLong(arg->ival)); + } + pArg = PyTuple_New(arg->len); + for(i = 0; i < arg->len; i++) { + PyTuple_SetItem(pArg, i, PyLong_FromLong(arg->iarray[i])); + } + return (pArg); + case DSPACES_ARG_STR: + return (PyUnicode_DecodeFSDefault(arg->strval)); + case DSPACES_ARG_NONE: + return (Py_None); + default: + fprintf(stderr, "ERROR: unknown arg type in %s (%d)\n", __func__, + arg->type); + } + + return (NULL); +} + +static struct dspaces_module_ret *py_res_buf(PyObject *pResult) +{ + PyArrayObject *pArray; + struct dspaces_module_ret *ret = malloc(sizeof(*ret)); + size_t data_len; + int flags; + npy_intp *dims; + int i; + + pArray = (PyArrayObject *)pResult; + flags = PyArray_FLAGS(pArray); + if(flags & NPY_ARRAY_C_CONTIGUOUS) { + ret->st = row_major; + } else if(flags & NPY_ARRAY_F_CONTIGUOUS) { + ret->st = column_major; + } else { + fprintf(stderr, "WARNING: bad array alignment in %s\n", __func__); + return (NULL); + } + ret->ndim = PyArray_NDIM(pArray); + ret->dim = malloc(sizeof(*ret->dim * ret->ndim)); + dims = PyArray_DIMS(pArray); + ret->len = 1; + for(i = 0; i < ret->ndim; i++) { + ret->dim[ret->st == column_major ? i : (ret->ndim - i) - 1] = dims[i]; + ret->len *= dims[i]; + } + ret->tag = PyArray_TYPE(pArray); + ret->elem_size = PyArray_ITEMSIZE(pArray); + data_len = ret->len * ret->elem_size; + ret->data = malloc(data_len); + + memcpy(ret->data, PyArray_DATA(pArray), data_len); + + ret->type = DSPACES_MOD_RET_ARRAY; + + return (ret); +} + +static struct dspaces_module_ret *py_res_int(PyObject *pResult) +{ + struct dspaces_module_ret *ret = malloc(sizeof(*ret)); + + ret->type = DSPACES_MOD_RET_INT; + ret->ival = PyLong_AsLong(pResult); + + return (ret); +} + +static struct dspaces_module_ret *py_res_to_ret(PyObject *pResult, int ret_type) +{ + struct dspaces_module_ret *ret; + + if(pResult == Py_None) { + return (NULL); + } + + switch(ret_type) { + case DSPACES_MOD_RET_ARRAY: + return (py_res_buf(pResult)); + case DSPACES_MOD_RET_INT: + return (py_res_int(pResult)); + case DSPACES_MOD_RET_NONE: + return (NULL); + default: + fprintf(stderr, "ERROR: unknown module return type in %s (%d)\n", + __func__, ret_type); + return (NULL); + } +} + +static struct dspaces_module_ret * +dspaces_module_py_exec(struct dspaces_module *mod, const char *operation, + struct dspaces_module_args *args, int nargs, + int ret_type) +{ + PyGILState_STATE gstate; + PyObject *pFunc; + PyObject *pKey, *pArg, *pArgs, *pKWArgs; + PyObject *pResult; + struct dspaces_module_ret *ret; + int i; + + DEBUG_OUT("calling out to '%s' in module '%s'.\n", operation, mod->name); + + gstate = PyGILState_Ensure(); + if(!mod->pModule) { + fprintf(stderr, + "ERROR: trying to run against failed Python module '%s'." + " Check for warnings from module load time.\n", + mod->name); + return (ds_module_ret_err(DS_MOD_ENOMOD)); + } + + pFunc = PyObject_GetAttrString(mod->pModule, operation); + if(!pFunc || !PyCallable_Check(pFunc)) { + fprintf( + stderr, + "ERROR! Could not find executable function '%s' in module '%s'\n", + operation, mod->name); + return (ds_module_ret_err(DS_MOD_ENODEF)); + } + pArgs = PyTuple_New(0); + pKWArgs = PyDict_New(); + for(i = 0; i < nargs; i++) { + pKey = PyUnicode_DecodeFSDefault(args[i].name); + pArg = py_obj_from_arg(&args[i]); + PyDict_SetItem(pKWArgs, pKey, pArg); + Py_DECREF(pKey); + Py_DECREF(pArg); + } + pResult = PyObject_Call(pFunc, pArgs, pKWArgs); + if(!pResult) { + PyErr_Print(); + ret = ds_module_ret_err(DS_MOD_EFAULT); + } else { + ret = py_res_to_ret(pResult, ret_type); + Py_DECREF(pResult); + } + + Py_DECREF(pArgs); + Py_DECREF(pKWArgs); + Py_DECREF(pFunc); + + PyGILState_Release(gstate); + + return (ret); +} +#endif // DSPACES_HAVE_PYTHON + +struct dspaces_module_ret *dspaces_module_exec(struct dspaces_module *mod, + const char *operation, + struct dspaces_module_args *args, + int nargs, int ret_type) +{ + if(mod->type == DSPACES_MOD_PY) { +#ifdef DSPACES_HAVE_PYTHON + return (dspaces_module_py_exec(mod, operation, args, nargs, ret_type)); +#else + fprintf(stderr, "WARNNING: tried to execute python module, but no " + "python support.\n"); + return (ds_module_ret_err(DS_MOD_ENOSUPPORT)); +#endif // DSPACES_HAVE_PYTHON + } else { + fprintf(stderr, "ERROR: unknown module request in %s.\n", __func__); + return (ds_module_ret_err(DS_MOD_ENOSYS)); + } +} diff --git a/src/dspaces-ops.c b/src/dspaces-ops.c index a92a1e68..187a9763 100644 --- a/src/dspaces-ops.c +++ b/src/dspaces-ops.c @@ -70,6 +70,8 @@ struct ds_data_expr *dspaces_op_new_iconst(long val) expr->type = DS_VAL_INT; expr->size = sizeof(int); expr->sub_expr = NULL; + + return (expr); } struct ds_data_expr *dspaces_op_new_rconst(double val) @@ -81,6 +83,8 @@ struct ds_data_expr *dspaces_op_new_rconst(double val) expr->type = DS_VAL_REAL; expr->size = sizeof(double); expr->sub_expr = NULL; + + return (NULL); } struct ds_data_expr *dspaces_op_new_add(struct ds_data_expr *expr1, diff --git a/src/dspaces-server.c b/src/dspaces-server.c index 10c990dd..4a99d32c 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -5,7 +5,11 @@ * See COPYRIGHT in top-level directory. */ #include "dspaces-server.h" +#include "dspaces-conf.h" +#include "dspaces-logging.h" +#include "dspaces-modules.h" #include "dspaces-ops.h" +#include "dspaces-remote.h" #include "dspaces-storage.h" #include "dspaces.h" #include "dspacesp.h" @@ -17,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -39,18 +44,6 @@ #include #endif /* HAVE_DRC */ -#define DEBUG_OUT(dstr, ...) \ - do { \ - if(server->f_debug) { \ - ABT_unit_id tid; \ - ABT_thread_self_id(&tid); \ - fprintf(stderr, \ - "Rank %i: TID: %" PRIu64 " %s, line %i (%s): " dstr, \ - server->rank, tid, __FILE__, __LINE__, __func__, \ - ##__VA_ARGS__); \ - } \ - } while(0); - #define DSPACES_DEFAULT_NUM_HANDLERS 4 #define xstr(s) str(s) @@ -68,52 +61,8 @@ struct addr_list_entry { char *addr; }; -struct remote { - char *name; - char addr_str[128]; - dspaces_client_t conn; -}; - -#define DSPACES_ARG_REAL 0 -#define DSPACES_ARG_INT 1 -#define DSPACES_ARG_STR 2 -#define DSPACES_ARG_NONE 3 -struct dspaces_module_args { - char *name; - int type; - int len; - union { - double rval; - long ival; - double *rarray; - long *iarray; - char *strval; - }; -}; - -#define DSPACES_MOD_RET_ARRAY 0 -struct dspaces_module_ret { - int type; - int len; - uint64_t *dim; - int ndim; - int tag; - int elem_size; - void *data; -}; - -#define DSPACES_MOD_PY 0 -struct dspaces_module { - char *name; - int type; - union { -#ifdef DSPACES_HAVE_PYTHON - PyObject *pModule; -#endif // DSPACES_HAVE_PYTHON - }; -}; - struct dspaces_provider { + struct ds_conf conf; struct list_head dirs; margo_instance_id mid; hg_id_t put_id; @@ -134,11 +83,12 @@ struct dspaces_provider { hg_id_t notify_id; hg_id_t do_ops_id; hg_id_t pexec_id; + hg_id_t mpexec_id; hg_id_t cond_id; hg_id_t get_vars_id; hg_id_t get_var_objs_id; - int nmods; - struct dspaces_module *mods; + hg_id_t reg_id; + struct list_head mods; struct ds_gspace *dsg; char **server_address; char **node_names; @@ -170,8 +120,19 @@ struct dspaces_provider { struct remote *remotes; int nremote; + + int num_handlers; + int *handler_rmap; + + atomic_int local_reg_id; +#ifdef DSPACES_HAVE_PYTHON + PyThreadState *main_state; + PyThreadState **handler_state; +#endif // DSPACES_HAVE_PYTHON }; +static int dspaces_init_registry(dspaces_provider_t server); + DECLARE_MARGO_RPC_HANDLER(put_rpc) DECLARE_MARGO_RPC_HANDLER(put_local_rpc) DECLARE_MARGO_RPC_HANDLER(put_meta_rpc) @@ -187,296 +148,36 @@ DECLARE_MARGO_RPC_HANDLER(sub_rpc) DECLARE_MARGO_RPC_HANDLER(do_ops_rpc) #ifdef DSPACES_HAVE_PYTHON DECLARE_MARGO_RPC_HANDLER(pexec_rpc) +DECLARE_MARGO_RPC_HANDLER(mpexec_rpc) #endif // DSPACES_HAVE_PYTHON DECLARE_MARGO_RPC_HANDLER(cond_rpc) DECLARE_MARGO_RPC_HANDLER(get_vars_rpc); DECLARE_MARGO_RPC_HANDLER(get_var_objs_rpc); - -static void put_rpc(hg_handle_t h); -static void put_local_rpc(hg_handle_t h); -static void put_meta_rpc(hg_handle_t h); -static void get_rpc(hg_handle_t h); -static void query_rpc(hg_handle_t h); -static void query_meta_rpc(hg_handle_t h); -static void obj_update_rpc(hg_handle_t h); -static void odsc_internal_rpc(hg_handle_t h); -static void ss_rpc(hg_handle_t h); -static void kill_rpc(hg_handle_t h); -static void sub_rpc(hg_handle_t h); -static void do_ops_rpc(hg_handle_t h); -static void pexec_rpc(hg_handle_t h); -static void cond_rpc(hg_handle_t h); -static void get_vars_rpc(hg_handle_t h); -static void get_vars_obj_rpc(hg_handle_t h); - -/* Server configuration parameters */ -static struct { - int ndim; - struct coord dims; - int max_versions; - int hash_version; /* 1 - ssd_hash_version_v1, 2 - ssd_hash_version_v2 */ - int num_apps; -} ds_conf; - -static struct { - const char *opt; - int *pval; -} options[] = {{"ndim", &ds_conf.ndim}, - {"dims", (int *)&ds_conf.dims}, - {"max_versions", &ds_conf.max_versions}, - {"hash_version", &ds_conf.hash_version}, - {"num_apps", &ds_conf.num_apps}}; - -static void eat_spaces(char *line) -{ - char *t = line; - - while(t && *t) { - if(*t != ' ' && *t != '\t' && *t != '\n') - *line++ = *t; - t++; - } - if(line) - *line = '\0'; -} - -static int parse_line(int lineno, char *line) -{ - char *t; - int i, n; - - /* Comment line ? */ - if(line[0] == '#') - return 0; - - t = strstr(line, "="); - if(!t) { - eat_spaces(line); - if(strlen(line) == 0) - return 0; - else - return -EINVAL; - } - - t[0] = '\0'; - eat_spaces(line); - t++; - - n = sizeof(options) / sizeof(options[0]); - - for(i = 0; i < n; i++) { - if(strcmp(line, options[1].opt) == 0) { /**< when "dims" */ - // get coordinates - int idx = 0; - char *crd; - crd = strtok(t, ","); - while(crd != NULL) { - ((struct coord *)options[1].pval)->c[idx] = atoll(crd); - crd = strtok(NULL, ","); - idx++; - } - if(idx != *(int *)options[0].pval) { - fprintf(stderr, "ERROR: (%s): dimensionality mismatch.\n", - __func__); - fprintf(stderr, "ERROR: index=%d, ndims=%d\n", idx, - *(int *)options[0].pval); - return -EINVAL; - } - break; - } - if(strcmp(line, options[i].opt) == 0) { - eat_spaces(line); - *(int *)options[i].pval = atoi(t); - break; - } - } - - if(i == n) { - fprintf(stderr, "WARNING: (%s): unknown option '%s' at line %d.\n", - __func__, line, lineno); - } - return 0; -} - -static int parse_conf(const char *fname) -{ - FILE *fin; - char buff[1024]; - int lineno = 1, err; - - fin = fopen(fname, "rt"); - if(!fin) { - fprintf(stderr, "ERROR: could not open configuration file '%s'.\n", - fname); - return -errno; - } - - while(fgets(buff, sizeof(buff), fin) != NULL) { - err = parse_line(lineno++, buff); - if(err < 0) { - fclose(fin); - return err; - } - } - - fclose(fin); - return 0; -} - -static int parse_conf_toml(const char *fname, struct list_head *dir_list, - struct remote **rem_array, int *nremote) -{ - FILE *fin; - toml_table_t *conf; - toml_table_t *storage; - toml_table_t *conf_dir; - toml_table_t *server; - toml_table_t *remotes; - toml_table_t *remote; - toml_datum_t dat; - toml_array_t *arr; - struct dspaces_dir *dir; - struct dspaces_file *file; - char errbuf[200]; - char *ip; - int port; - int ndim = 0; - int ndir, nfile; - int i, j, n; - - fin = fopen(fname, "r"); - if(!fin) { - fprintf(stderr, "ERROR: could not open configuration file '%s'.\n", - fname); - return -errno; - } - - conf = toml_parse_file(fin, errbuf, sizeof(errbuf)); - fclose(fin); - - if(!conf) { - fprintf(stderr, "could not parse %s, %s.\n", fname, errbuf); - return -1; - } - - server = toml_table_in(conf, "server"); - if(!server) { - fprintf(stderr, "missing [server] block from %s\n", fname); - return -1; - } - - n = sizeof(options) / sizeof(options[0]); - for(i = 0; i < n; i++) { - if(strcmp(options[i].opt, "dims") == 0) { - arr = toml_array_in(server, "dims"); - if(arr) { - while(1) { - dat = toml_int_at(arr, ndim); - if(!dat.ok) { - break; - } - ((struct coord *)options[i].pval)->c[ndim] = dat.u.i; - ndim++; - } - } - for(j = 0; j < n; j++) { - if(strcmp(options[j].opt, "ndim") == 0) { - *(int *)options[j].pval = ndim; - } - } - } else { - dat = toml_int_in(server, options[i].opt); - if(dat.ok) { - *(int *)options[i].pval = dat.u.i; - } - } - } - - remotes = toml_table_in(conf, "remotes"); - if(remotes) { - *nremote = toml_table_ntab(remotes); - *rem_array = malloc(sizeof(**rem_array) * *nremote); - for(i = 0; i < *nremote; i++) { - // remote = toml_table_at(remotes, i); - (*rem_array)[i].name = strdup(toml_key_in(remotes, i)); - remote = toml_table_in(remotes, (*rem_array)[i].name); - dat = toml_string_in(remote, "ip"); - ip = dat.u.s; - dat = toml_int_in(remote, "port"); - port = dat.u.i; - sprintf((*rem_array)[i].addr_str, "sockets://%s:%i", ip, port); - free(ip); - } - } - - storage = toml_table_in(conf, "storage"); - if(storage) { - ndir = toml_table_ntab(storage); - for(i = 0; i < ndir; i++) { - dir = malloc(sizeof(*dir)); - dir->name = strdup(toml_key_in(storage, i)); - conf_dir = toml_table_in(storage, dir->name); - dat = toml_string_in(conf_dir, "directory"); - dir->path = strdup(dat.u.s); - free(dat.u.s); - if(0 != (arr = toml_array_in(conf_dir, "files"))) { - INIT_LIST_HEAD(&dir->files); - nfile = toml_array_nelem(arr); - for(j = 0; j < nfile; j++) { - dat = toml_string_at(arr, j); - file = malloc(sizeof(*file)); - file->type = DS_FILE_NC; - file->name = strdup(dat.u.s); - free(dat.u.s); - list_add(&file->entry, &dir->files); - } - } else { - dat = toml_string_in(conf_dir, "files"); - if(dat.ok) { - if(strcmp(dat.u.s, "all") == 0) { - dir->cont_type = DS_FILE_ALL; - } else { - fprintf(stderr, - "ERROR: %s: invalid value for " - "storage.%s.files: %s\n", - __func__, dir->name, dat.u.s); - } - free(dat.u.s); - } else { - fprintf(stderr, - "ERROR: %s: no readable 'files' key for '%s'.\n", - __func__, dir->name); - } - } - list_add(&dir->entry, dir_list); - } - } - - toml_free(conf); -} +DECLARE_MARGO_RPC_HANDLER(reg_rpc); static int init_sspace(dspaces_provider_t server, struct bbox *default_domain, struct ds_gspace *dsg_l) { int err = -ENOMEM; - dsg_l->ssd = ssd_alloc(default_domain, dsg_l->size_sp, ds_conf.max_versions, - ds_conf.hash_version); + dsg_l->ssd = + ssd_alloc(default_domain, dsg_l->size_sp, server->conf.max_versions, + server->conf.hash_version); if(!dsg_l->ssd) goto err_out; - if(ds_conf.hash_version == ssd_hash_version_auto) { - DEBUG_OUT("server selected hash version %i for default space\n", - dsg_l->ssd->hash_version); + if(server->conf.hash_version == ssd_hash_version_auto) { + DEBUG_OUT("server selected hash type %s for default space\n", + hash_strings[dsg_l->ssd->hash_version]); } err = ssd_init(dsg_l->ssd, dsg_l->rank); if(err < 0) goto err_out; - dsg_l->default_gdim.ndim = ds_conf.ndim; + dsg_l->default_gdim.ndim = server->conf.ndim; int i; - for(i = 0; i < ds_conf.ndim; i++) { - dsg_l->default_gdim.sizes.c[i] = ds_conf.dims.c[i]; + for(i = 0; i < server->conf.ndim; i++) { + dsg_l->default_gdim.sizes.c[i] = server->conf.dims.c[i]; } INIT_LIST_HEAD(&dsg_l->sspace_list); @@ -606,30 +307,6 @@ static int write_conf(dspaces_provider_t server, MPI_Comm comm) return (ret); } -const char *hash_strings[] = {"Dynamic", "SFC", "Bisection"}; - -void print_conf() -{ - int i; - - printf("DataSpaces server config:\n"); - printf("=========================\n"); - printf(" Default global dimensions: ("); - printf("%" PRIu64, ds_conf.dims.c[0]); - for(i = 1; i < ds_conf.ndim; i++) { - printf(", %" PRIu64, ds_conf.dims.c[i]); - } - printf(")\n"); - printf(" MAX STORED VERSIONS: %i\n", ds_conf.max_versions); - printf(" HASH TYPE: %s\n", hash_strings[ds_conf.hash_version]); - if(ds_conf.num_apps >= 0) { - printf(" APPS EXPECTED: %i\n", ds_conf.num_apps); - } else { - printf(" RUN UNTIL KILLED\n"); - } - printf("=========================\n"); -} - static int dsg_alloc(dspaces_provider_t server, const char *conf_name, MPI_Comm comm) { @@ -638,55 +315,61 @@ static int dsg_alloc(dspaces_provider_t server, const char *conf_name, int err = -ENOMEM; /* Default values */ - ds_conf.max_versions = 255; - ds_conf.hash_version = ssd_hash_version_auto; - ds_conf.num_apps = -1; + server->conf.max_versions = 255; + server->conf.hash_version = ssd_hash_version_auto; + server->conf.num_apps = -1; INIT_LIST_HEAD(&server->dirs); + INIT_LIST_HEAD(&server->mods); + + // TODO: distribute configuration from root. ext = strrchr(conf_name, '.'); if(!ext || strcmp(ext, ".toml") != 0) { - err = parse_conf(conf_name); + err = parse_conf(conf_name, &server->conf); } else { - err = parse_conf_toml(conf_name, &server->dirs, &server->remotes, - &server->nremote); + server->conf.dirs = &server->dirs; + server->conf.remotes = &server->remotes; + server->conf.mods = &server->mods; + err = parse_conf_toml(conf_name, &server->conf); + server->nremote = server->conf.nremote; } if(err < 0) { goto err_out; } // Check number of dimension - if(ds_conf.ndim > BBOX_MAX_NDIM) { + if(server->conf.ndim > BBOX_MAX_NDIM) { fprintf( stderr, "%s(): ERROR maximum number of array dimension is %d but ndim is %d" " in file '%s'\n", - __func__, BBOX_MAX_NDIM, ds_conf.ndim, conf_name); + __func__, BBOX_MAX_NDIM, server->conf.ndim, conf_name); err = -EINVAL; goto err_out; - } else if(ds_conf.ndim == 0) { + } else if(server->conf.ndim == 0) { DEBUG_OUT( "no global coordinates provided. Setting trivial placeholder.\n"); - ds_conf.ndim = 1; - ds_conf.dims.c[0] = 1; + server->conf.ndim = 1; + server->conf.dims.c[0] = 1; } // Check hash version - if((ds_conf.hash_version < ssd_hash_version_auto) || - (ds_conf.hash_version >= _ssd_hash_version_count)) { + if((server->conf.hash_version < ssd_hash_version_auto) || + (server->conf.hash_version >= _ssd_hash_version_count)) { fprintf(stderr, "%s(): ERROR unknown hash version %d in file '%s'\n", - __func__, ds_conf.hash_version, conf_name); + __func__, server->conf.hash_version, conf_name); err = -EINVAL; goto err_out; } struct bbox domain; memset(&domain, 0, sizeof(struct bbox)); - domain.num_dims = ds_conf.ndim; + domain.num_dims = server->conf.ndim; int i; for(i = 0; i < domain.num_dims; i++) { domain.lb.c[i] = 0; - domain.ub.c[i] = ds_conf.dims.c[i] - 1; + domain.ub.c[i] = server->conf.dims.c[i] - 1; } dsg_l = malloc(sizeof(*dsg_l)); @@ -698,7 +381,7 @@ static int dsg_alloc(dspaces_provider_t server, const char *conf_name, MPI_Comm_rank(comm, &dsg_l->rank); if(dsg_l->rank == 0) { - print_conf(); + print_conf(&server->conf); } write_conf(server, comm); @@ -707,20 +390,20 @@ static int dsg_alloc(dspaces_provider_t server, const char *conf_name, if(err < 0) { goto err_free; } - dsg_l->ls = ls_alloc(ds_conf.max_versions); + dsg_l->ls = ls_alloc(server->conf.max_versions); if(!dsg_l->ls) { fprintf(stderr, "%s(): ERROR ls_alloc() failed\n", __func__); goto err_free; } // proxy storage - dsg_l->ps = ls_alloc(ds_conf.max_versions); + dsg_l->ps = ls_alloc(server->conf.max_versions); if(!dsg_l->ps) { fprintf(stderr, "%s(): ERROR ls_alloc() failed\n", __func__); goto err_free; } - dsg_l->num_apps = ds_conf.num_apps; + dsg_l->num_apps = server->conf.num_apps; INIT_LIST_HEAD(&dsg_l->obj_desc_drain_list); @@ -805,15 +488,16 @@ static struct sspace *lookup_sspace(dspaces_provider_t server, memcpy(&ssd_entry->gdim, &gdim, sizeof(struct global_dimension)); DEBUG_OUT("allocate the ssd.\n"); - ssd_entry->ssd = ssd_alloc(&domain, dsg_l->size_sp, ds_conf.max_versions, - ds_conf.hash_version); + ssd_entry->ssd = + ssd_alloc(&domain, dsg_l->size_sp, server->conf.max_versions, + server->conf.hash_version); if(!ssd_entry->ssd) { fprintf(stderr, "%s(): ssd_alloc failed for '%s'\n", __func__, var_name); return dsg_l->ssd; } - if(ds_conf.hash_version == ssd_hash_version_auto) { + if(server->conf.hash_version == ssd_hash_version_auto) { DEBUG_OUT("server selected hash version %i for var %s\n", ssd_entry->ssd->hash_version, var_name); } @@ -924,7 +608,7 @@ static int get_client_data(obj_descriptor odsc, dspaces_provider_t server) in.odsc.raw_odsc = (char *)(&odsc); hg_addr_t owner_addr; - size_t owner_addr_size = 128; + hg_size_t owner_addr_size = 128; margo_addr_self(server->mid, &owner_addr); margo_addr_to_string(server->mid, od->obj_desc.owner, &owner_addr_size, @@ -1007,32 +691,18 @@ static void drain_thread(void *arg) } #ifdef DSPACES_HAVE_PYTHON -static void *bootstrap_python() -{ - Py_Initialize(); - import_array(); -} - -static int dspaces_init_py_mods(dspaces_provider_t server, - struct dspaces_module **pmodsp) +static void *bootstrap_python(dspaces_provider_t server) { - static const char *static_pmods[][2] = { - {"goes17", "s3nc_mod"}, - {"planetary", "azure_mod"}, - {"cmips3", "s3cmip_mod"} - }; char *pypath = getenv("PYTHONPATH"); char *new_pypath; int pypath_len; - struct dspaces_module *pmods; - int npmods = sizeof(static_pmods) / sizeof(static_pmods[0]); - PyObject *pName; int i; pypath_len = strlen(xstr(DSPACES_MOD_DIR)) + 1; if(pypath) { pypath_len += strlen(pypath) + 1; } + new_pypath = malloc(pypath_len); if(pypath) { sprintf(new_pypath, "%s:%s", xstr(DSPACES_MOD_DIR), pypath); @@ -1042,35 +712,51 @@ static int dspaces_init_py_mods(dspaces_provider_t server, setenv("PYTHONPATH", new_pypath, 1); DEBUG_OUT("New PYTHONPATH is %s\n", new_pypath); - bootstrap_python(); - - pmods = malloc(sizeof(*pmods) * npmods); - for(i = 0; i < npmods; i++) { - pmods[i].name = strdup(static_pmods[i][0]); - pmods[i].type = DSPACES_MOD_PY; - pName = PyUnicode_DecodeFSDefault(static_pmods[i][1]); - pmods[i].pModule = PyImport_Import(pName); - if(pmods[0].pModule == NULL) { - fprintf(stderr, - "WARNING: could not load %s mod from %s. File missing? Any " - "%s accesses will fail.\n", - static_pmods[i][1], xstr(DSPACES_MOD_DIR), pmods[i].name); - } - Py_DECREF(pName); - } - free(new_pypath); + Py_InitializeEx(0); + import_array(); - *pmodsp = pmods; + server->handler_state = + malloc(sizeof(*server->handler_state) * server->num_handlers); - return (npmods); + return (NULL); } + #endif // DSPACES_HAVE_PYTHON -void dspaces_init_mods(dspaces_provider_t server) +static void init_rmap(struct dspaces_provider *server) { -#ifdef DSPACES_HAVE_PYTHON - server->nmods = dspaces_init_py_mods(server, &server->mods); -#endif // DSPACES_HAVE_PYTHON + int i; + + server->handler_rmap = + malloc(sizeof(*server->handler_rmap) * server->num_handlers); + for(i = 0; i < server->num_handlers; i++) { + server->handler_rmap[i] = -1; + } +} + +static int get_handler_id(struct dspaces_provider *server) +{ + int es_rank; + int i; + + ABT_self_get_xstream_rank(&es_rank); + + for(i = 0; i < server->num_handlers; i++) { + if(server->handler_rmap[i] == -1) { + server->handler_rmap[i] = es_rank; + } + + if(server->handler_rmap[i] == es_rank) { + return (i); + } + } + + fprintf(stderr, + "ERROR: more unique execution streams have called %s than have " + "been allocated for RPC handlers.\n", + __func__); + + return (-1); } int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, @@ -1079,12 +765,12 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, const char *envdebug = getenv("DSPACES_DEBUG"); const char *envnthreads = getenv("DSPACES_NUM_HANDLERS"); const char *envdrain = getenv("DSPACES_DRAIN"); + const char *mod_dir_str = xstr(DSPACES_MOD_DIR); dspaces_provider_t server; hg_class_t *hg; static int is_initialized = 0; hg_bool_t flag; hg_id_t id; - int num_handlers = DSPACES_DEFAULT_NUM_HANDLERS; struct hg_init_info hii = HG_INIT_INFO_INITIALIZER; char margo_conf[1024]; struct margo_init_info mii = MARGO_INIT_INFO_INITIALIZER; @@ -1106,9 +792,11 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, server->f_debug = 1; } + server->num_handlers = DSPACES_DEFAULT_NUM_HANDLERS; if(envnthreads) { - num_handlers = atoi(envnthreads); + server->num_handlers = atoi(envnthreads); } + init_rmap(server); if(envdrain) { DEBUG_OUT("enabling data draining.\n"); @@ -1118,15 +806,12 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, MPI_Comm_dup(comm, &server->comm); MPI_Comm_rank(comm, &server->rank); - dspaces_init_mods(server); - - const char *mod_dir_str = xstr(DSPACES_MOD_DIR); - DEBUG_OUT("module directory is %s\n", mod_dir_str); + dspaces_init_logging(server->rank); margo_set_environment(NULL); sprintf(margo_conf, "{ \"use_progress_thread\" : true, \"rpc_thread_count\" : %d }", - num_handlers); + server->num_handlers); hii.request_post_init = 1024; hii.auto_sm = false; mii.hg_init_info = &hii; @@ -1266,6 +951,10 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, &flag); DS_HG_REGISTER(hg, server->pexec_id, pexec_in_t, pexec_out_t, pexec_rpc); + margo_registered_name(server->mid, "mpexec_rpc", &server->mpexec_id, + &flag); + DS_HG_REGISTER(hg, server->mpexec_id, pexec_in_t, pexec_out_t, + pexec_rpc); #endif // DSPACES_HAVE_PYTHON margo_registered_name(server->mid, "cond_rpc", &server->cond_id, &flag); DS_HG_REGISTER(hg, server->cond_id, cond_in_t, void, cond_rpc); @@ -1277,6 +966,8 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, &server->get_var_objs_id, &flag); DS_HG_REGISTER(hg, server->get_var_objs_id, get_var_objs_in_t, odsc_hdr, get_var_objs_rpc); + margo_registered_name(server->mid, "reg_rpc", &server->reg_id, &flag); + DS_HG_REGISTER(hg, server->reg_id, reg_in_t, uint64_t, reg_rpc); } else { server->put_id = MARGO_REGISTER(server->mid, "put_rpc", bulk_gdim_t, bulk_out_t, put_rpc); @@ -1352,6 +1043,10 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, pexec_out_t, pexec_rpc); margo_register_data(server->mid, server->pexec_id, (void *)server, NULL); + server->mpexec_id = MARGO_REGISTER(server->mid, "mpexec_rpc", + pexec_in_t, pexec_out_t, mpexec_rpc); + margo_register_data(server->mid, server->mpexec_id, (void *)server, + NULL); #endif // DSPACES_HAVE_PYTHON server->cond_id = MARGO_REGISTER(server->mid, "cond_rpc", cond_in_t, void, cond_rpc); @@ -1367,6 +1062,9 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, odsc_hdr, get_var_objs_rpc); margo_register_data(server->mid, server->get_var_objs_id, (void *)server, NULL); + server->reg_id = + MARGO_REGISTER(server->mid, "reg_rpc", reg_in_t, uint64_t, reg_rpc); + margo_register_data(server->mid, server->reg_id, (void *)server, NULL); } int err = dsg_alloc(server, conf_file, comm); if(err) { @@ -1391,6 +1089,17 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, DEBUG_OUT("Server will run indefinitely.\n"); } +#ifdef DSPACES_HAVE_PYTHON + bootstrap_python(server); +#endif // DSPACES_HAVE_PYTHON + + DEBUG_OUT("module directory is %s\n", mod_dir_str); + dspaces_init_mods(&server->mods); + dspaces_init_registry(server); +#ifdef DSPACES_HAVE_PYTHON + server->main_state = PyEval_SaveThread(); +#endif // DSPACES_HAVE_PYTHON + if(server->f_drain) { // thread to drain the data ABT_xstream_create(ABT_SCHED_NULL, &server->drain_xstream); @@ -1415,6 +1124,8 @@ int dspaces_server_init(const char *listen_addr_str, MPI_Comm comm, is_initialized = 1; + DEBUG_OUT("server is ready for requests.\n"); + return dspaces_SUCCESS; } @@ -1541,7 +1252,7 @@ static void address_translate(dspaces_provider_t server, char *addr_str) static void odsc_take_ownership(dspaces_provider_t server, obj_descriptor *odsc) { hg_addr_t owner_addr; - size_t owner_addr_size = 128; + hg_size_t owner_addr_size = 128; margo_addr_self(server->mid, &owner_addr); margo_addr_to_string(server->mid, odsc->owner, &owner_addr_size, @@ -1596,7 +1307,7 @@ static void put_rpc(hg_handle_t handle) hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); - DEBUG_OUT("Creating a bulk transfer buffer of size %li\n", size); + DEBUG_OUT("Creating a bulk transfer buffer of size %" PRIu64 "\n", size); hret = margo_bulk_create(mid, 1, (void **)&(od->data), &size, HG_BULK_WRITE_ONLY, &bulk_handle); @@ -1810,7 +1521,7 @@ static int query_remotes(dspaces_provider_t server, obj_descriptor *q_odsc, obj_descriptor *odsc; struct obj_data *od; hg_addr_t owner_addr; - size_t owner_addr_size = 128; + hg_size_t owner_addr_size = 128; int i; buf_size = obj_data_size(q_odsc); @@ -1858,274 +1569,30 @@ static int query_remotes(dspaces_provider_t server, obj_descriptor *q_odsc, } } -struct dspaces_module *dspaces_find_mod(dspaces_provider_t server, - const char *mod_name) -{ - int i; - - for(i = 0; i < server->nmods; i++) { - if(strcmp(mod_name, server->mods[i].name) == 0) { - return (&server->mods[i]); - } - } - - return (NULL); -} - -#ifdef DSPACES_HAVE_PYTHON -PyObject *py_obj_from_arg(struct dspaces_module_args *arg) -{ - PyObject *pArg; - int i; - - switch(arg->type) { - case DSPACES_ARG_REAL: - if(arg->len == -1) { - return (PyFloat_FromDouble(arg->rval)); - } - pArg = PyTuple_New(arg->len); - for(i = 0; i < arg->len; i++) { - PyTuple_SetItem(pArg, i, PyFloat_FromDouble(arg->rarray[i])); - } - return (pArg); - case DSPACES_ARG_INT: - if(arg->len == -1) { - return (PyLong_FromLong(arg->ival)); - } - pArg = PyTuple_New(arg->len); - for(i = 0; i < arg->len; i++) { - PyTuple_SetItem(pArg, i, PyLong_FromLong(arg->iarray[i])); - } - return (pArg); - case DSPACES_ARG_STR: - return (PyUnicode_DecodeFSDefault(arg->strval)); - case DSPACES_ARG_NONE: - return (Py_None); - default: - fprintf(stderr, "ERROR: unknown arg type in %s (%d)\n", __func__, - arg->type); - } - - return (NULL); -} - -static struct dspaces_module_ret *py_res_buf(PyObject *pResult) -{ - PyArrayObject *pArray; - struct dspaces_module_ret *ret = malloc(sizeof(*ret)); - size_t data_len; - npy_intp *dims; - int i; - - pArray = (PyArrayObject *)pResult; - ret->ndim = PyArray_NDIM(pArray); - ret->dim = malloc(sizeof(*ret->dim * ret->ndim)); - dims = PyArray_DIMS(pArray); - ret->len = 1; - for(i = 0; i < ret->ndim; i++) { - ret->dim[i] = dims[i]; - ret->len *= dims[i]; - } - ret->tag = PyArray_TYPE(pArray); - ret->elem_size = PyArray_ITEMSIZE(pArray); - data_len = ret->len * ret->elem_size; - ret->data = malloc(data_len); - memcpy(ret->data, PyArray_DATA(pArray), data_len); - - return (ret); -} - -static struct dspaces_module_ret *py_res_to_ret(PyObject *pResult, int ret_type) -{ - struct dspaces_module_ret *ret; - - switch(ret_type) { - case DSPACES_MOD_RET_ARRAY: - return (py_res_buf(pResult)); - default: - fprintf(stderr, "ERROR: unknown module return type in %s (%d)\n", - __func__, ret_type); - return (NULL); - } -} - -static struct dspaces_module_ret * -dspaces_module_py_exec(dspaces_provider_t server, struct dspaces_module *mod, - const char *operation, struct dspaces_module_args *args, - int nargs, int ret_type) -{ - PyObject *pFunc = PyObject_GetAttrString(mod->pModule, operation); - PyObject *pKey, *pArg, *pArgs, *pKWArgs; - PyObject *pResult; - struct dspaces_module_ret *ret; - int i; - - DEBUG_OUT("doing python module exec.\n"); - if(!pFunc || !PyCallable_Check(pFunc)) { - fprintf( - stderr, - "ERROR! Could not find executable function '%s' in module '%s'\n", - operation, mod->name); - return (NULL); - } - pArgs = PyTuple_New(0); - pKWArgs = PyDict_New(); - for(i = 0; i < nargs; i++) { - pKey = PyUnicode_DecodeFSDefault(args[i].name); - pArg = py_obj_from_arg(&args[i]); - PyDict_SetItem(pKWArgs, pKey, pArg); - Py_DECREF(pKey); - Py_DECREF(pArg); - } - pResult = PyObject_Call(pFunc, pArgs, pKWArgs); - if(!pResult) { - PyErr_Print(); - ret = NULL; - } else { - ret = py_res_to_ret(pResult, ret_type); - Py_DECREF(pResult); - } - - Py_DECREF(pArgs); - Py_DECREF(pKWArgs); - Py_DECREF(pFunc); - - return (ret); -} -#endif // DSPACES_HAVE_PYTHON - -static struct dspaces_module_ret * -dspaces_module_exec(dspaces_provider_t server, const char *mod_name, - const char *operation, struct dspaces_module_args *args, - int nargs, int ret_type) -{ - struct dspaces_module *mod = dspaces_find_mod(server, mod_name); - - DEBUG_OUT("sending '%s' to module '%s'\n", operation, mod->name); - if(mod->type == DSPACES_MOD_PY) { -#ifdef DSPACES_HAVE_PYTHON - return (dspaces_module_py_exec(server, mod, operation, args, nargs, - ret_type)); -#else - fprintf(stderr, "WARNNING: tried to execute python module, but not " - "python support.\n"); - return (NULL); -#endif // DSPACES_HAVE_PYTHON - } else { - fprintf(stderr, "ERROR: unknown module request in %s.\n", __func__); - return (NULL); - } -} - -static int build_module_args_from_odsc(obj_descriptor *odsc, - struct dspaces_module_args **argsp) -{ - struct dspaces_module_args *args; - int nargs = 4; - int ndims; - int i; - - args = malloc(sizeof(*args) * nargs); - - // odsc->name - args[0].name = strdup("name"); - args[0].type = DSPACES_ARG_STR; - args[0].len = strlen(odsc->name) + 1; - args[0].strval = strdup(odsc->name); - - // odsc->version - args[1].name = strdup("version"); - args[1].type = DSPACES_ARG_INT; - args[1].len = -1; - args[1].ival = odsc->version; - - ndims = odsc->bb.num_dims; - // odsc->bb.lb - args[2].name = strdup("lb"); - if(ndims > 0) { - args[2].name = strdup("lb"); - args[2].type = DSPACES_ARG_INT; - args[2].len = ndims; - args[2].iarray = malloc(sizeof(*args[2].iarray) * ndims); - for(i = 0; i < ndims; i++) { - args[2].iarray[i] = odsc->bb.lb.c[i]; - } - } else { - args[2].type = DSPACES_ARG_NONE; - } - - // odsc->bb.ub - args[3].name = strdup("ub"); - if(ndims > 0) { - args[3].type = DSPACES_ARG_INT; - args[3].len = ndims; - args[3].iarray = malloc(sizeof(*args[3].iarray) * ndims); - for(i = 0; i < ndims; i++) { - args[3].iarray[i] = odsc->bb.ub.c[i]; - } - } else { - args[3].type = DSPACES_ARG_NONE; - } - - *argsp = args; - return (nargs); -} - -static void free_arg(struct dspaces_module_args *arg) -{ - if(arg) { - free(arg->name); - if(arg->len > 0 && arg->type != DSPACES_ARG_NONE) { - free(arg->strval); - } - } else { - fprintf(stderr, "WARNING: trying to free NULL argument in %s\n", - __func__); - } -} - -static void free_arg_list(struct dspaces_module_args *args, int len) -{ - int i; - - if(args) { - for(i = 0; i < len; i++) { - free_arg(&args[i]); - } - } else if(len > 0) { - fprintf(stderr, "WARNING: trying to free NULL argument list in %s\n", - __func__); - } -} - -static void route_request(dspaces_provider_t server, obj_descriptor *odsc, - struct global_dimension *gdim) +static int route_request(dspaces_provider_t server, obj_descriptor *odsc, + struct global_dimension *gdim) { - static const char *module_nspaces[][2] = { - {"goes17\\", "goes17"}, - {"cmip6-planetary\\", "planetary"}, - {"cmip6-s3\\", "cmips3"}, - {NULL, NULL} - }; - const char *s3nc_nspace = "goes17\\"; - const char *azure_nspace = "cmip6-planetary\\"; - const char *s3cmip_nspace = "cmip6-s3\\"; + struct dspaces_module *mod; struct dspaces_module_args *args; struct dspaces_module_ret *res = NULL; struct obj_data *od; obj_descriptor *od_odsc; - char **mod_desc; int nargs; - int i; + int i, err; DEBUG_OUT("Routing '%s'\n", odsc->name); - for(mod_desc = (char **)module_nspaces[0]; mod_desc[0] != NULL; mod_desc+=2) { - if(strstr(odsc->name, mod_desc[0]) == odsc->name) { - nargs = build_module_args_from_odsc(odsc, &args); - res = dspaces_module_exec(server, mod_desc[1], "query", args, - nargs, DSPACES_MOD_RET_ARRAY); - } + mod = dspaces_mod_by_od(&server->mods, odsc); + if(mod) { + nargs = build_module_args_from_odsc(odsc, &args); + res = dspaces_module_exec(mod, "query", args, nargs, + DSPACES_MOD_RET_ARRAY); + } + + if(res && res->type == DSPACES_MOD_RET_ERR) { + err = res->err; + free(res); + return (err); } if(res) { @@ -2138,17 +1605,22 @@ static void route_request(dspaces_provider_t server, obj_descriptor *odsc, free(res->data); } else { odsc->size = res->elem_size; - if(odsc->bb.num_dims == 0) { + if(odsc->bb.num_dims != res->ndim) { + DEBUG_OUT("change in dimensionality.\n"); odsc->bb.num_dims = res->ndim; obj_data_resize(odsc, res->dim); } else if((odsc->size * res->len) != obj_data_size(odsc)) { DEBUG_OUT("returned data is cropped.\n"); obj_data_resize(odsc, res->dim); } + // TODO: refactor to convert ret->odsc in function od_odsc = malloc(sizeof(*od_odsc)); memcpy(od_odsc, odsc, sizeof(*od_odsc)); odsc_take_ownership(server, od_odsc); od_odsc->tag = res->tag; + if(odsc->tag == 0) { + odsc->tag = res->tag; + } od = obj_data_alloc_no_data(od_odsc, res->data); memcpy(&od->gdim, gdim, sizeof(struct global_dimension)); DEBUG_OUT("adding object to local storage: %s\n", @@ -2161,6 +1633,8 @@ static void route_request(dspaces_provider_t server, obj_descriptor *odsc, } free(res); } + + return (0); } // should we handle this locally @@ -2169,41 +1643,129 @@ static int local_responsibility(dspaces_provider_t server, obj_descriptor *odsc) return (!server->remotes || ls_lookup(server->dsg->ls, odsc->name)); } -static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, - int timeout, obj_descriptor **results, int req_id) +static void send_tgt_rpc(dspaces_provider_t server, hg_id_t rpc_id, int target, + void *in, hg_addr_t *addr, hg_handle_t *h, + margo_request *req) { - struct sspace *ssd; - struct dht_entry **de_tab; - int peer_num; - int self_id_num = -1; - int total_odscs = 0; - int *odsc_nums; - obj_descriptor **odsc_tabs, **podsc = NULL; - obj_descriptor *odsc_curr; - margo_request *serv_reqs; - hg_handle_t *hndls; - hg_addr_t server_addr; - odsc_list_t dht_resp; - obj_descriptor *q_odsc; - struct global_dimension *q_gdim; - int dup; - int i, j, k; + DEBUG_OUT("sending rpc_id %" PRIu64 " to rank %i\n", rpc_id, target); + margo_addr_lookup(server->mid, server->server_address[target], addr); + margo_create(server->mid, *addr, rpc_id, h); + margo_iforward(*h, in, req); +} - q_odsc = (obj_descriptor *)query->odsc_gdim.raw_odsc; - q_gdim = (struct global_dimension *)query->odsc_gdim.raw_gdim; +struct ibcast_state { + int sent_rpc[3]; + hg_addr_t addr[3]; + hg_handle_t hndl[3]; + margo_request req[3]; +}; - if(!local_responsibility(server, q_odsc)) { - DEBUG_OUT("req %i: no local objects with name %s. Checking remotes.\n", - req_id, q_odsc->name); - return ( - query_remotes(server, (obj_descriptor *)query->odsc_gdim.raw_odsc, - (struct global_dimension *)query->odsc_gdim.raw_gdim, - timeout, results, req_id)); - } +static struct ibcast_state *ibcast_rpc_start(dspaces_provider_t server, + hg_id_t rpc_id, int32_t src, + void *in) +{ + struct ibcast_state *bcast; + int32_t rank, parent, child1, child2; - DEBUG_OUT("getting sspace lock.\n"); - ABT_mutex_lock(server->sspace_mutex); - DEBUG_OUT("got lock, looking up shared space for global dimensions.\n"); + rank = server->dsg->rank; + parent = (rank - 1) / 2; + child1 = (rank * 2) + 1; + child2 = child1 + 1; + + if(server->dsg->size_sp == 1 || + (src == parent && child1 >= server->dsg->size_sp)) { + return NULL; + } + + bcast = calloc(1, sizeof(*bcast)); + + if((src == -1 || src > rank) && rank > 0) { + DEBUG_OUT("sending to parent (%i)\n", parent); + send_tgt_rpc(server, rpc_id, parent, in, &bcast->addr[0], + &bcast->hndl[0], &bcast->req[0]); + bcast->sent_rpc[0] = 1; + } + if((child1 != src && child1 < server->dsg->size_sp)) { + DEBUG_OUT("sending to child 1 (%i)\n", child1); + send_tgt_rpc(server, rpc_id, child1, in, &bcast->addr[1], + &bcast->hndl[1], &bcast->req[1]); + bcast->sent_rpc[1] = 1; + } + if((child2 != src && child2 < server->dsg->size_sp)) { + DEBUG_OUT("sending to child 2 (%i)\n", child2); + send_tgt_rpc(server, rpc_id, child2, in, &bcast->addr[2], + &bcast->hndl[2], &bcast->req[2]); + bcast->sent_rpc[2] = 1; + } + + return (bcast); +} + +static void ibcast_get_output(struct ibcast_state *bcast, unsigned int idx, + void *out, int *present) +{ + if(idx > 2 || !bcast->sent_rpc[idx]) { + if(present) { + *present = 0; + } + return; + } + + margo_wait(bcast->req[idx]); + margo_get_output(bcast->hndl[idx], out); + if(present) { + *present = 1; + } +} + +static void ibcast_finish(dspaces_provider_t server, struct ibcast_state *bcast) +{ + int i; + + for(i = 0; i < 3; i++) { + if(bcast->sent_rpc[i]) { + margo_addr_free(server->mid, bcast->addr[i]); + margo_destroy(bcast->hndl[i]); + } + } + free(bcast); +} + +static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, + int timeout, obj_descriptor **results, int req_id) +{ + struct sspace *ssd; + struct dht_entry **de_tab; + int peer_num; + int self_id_num = -1; + int total_odscs = 0; + int *odsc_nums; + obj_descriptor **odsc_tabs, **podsc = NULL; + obj_descriptor *odsc_curr; + margo_request *serv_reqs; + hg_handle_t *hndls; + hg_addr_t server_addr; + odsc_list_t dht_resp; + obj_descriptor *q_odsc; + struct global_dimension *q_gdim; + int dup; + int i, j, k; + + q_odsc = (obj_descriptor *)query->odsc_gdim.raw_odsc; + q_gdim = (struct global_dimension *)query->odsc_gdim.raw_gdim; + + if(!local_responsibility(server, q_odsc)) { + DEBUG_OUT("req %i: no local objects with name %s. Checking remotes.\n", + req_id, q_odsc->name); + return ( + query_remotes(server, (obj_descriptor *)query->odsc_gdim.raw_odsc, + (struct global_dimension *)query->odsc_gdim.raw_gdim, + timeout, results, req_id)); + } + + DEBUG_OUT("getting sspace lock.\n"); + ABT_mutex_lock(server->sspace_mutex); + DEBUG_OUT("got lock, looking up shared space for global dimensions.\n"); ssd = lookup_sspace(server, q_odsc->name, q_gdim); ABT_mutex_unlock(server->sspace_mutex); DEBUG_OUT("found shared space with %i entries.\n", ssd->dht->num_entries); @@ -2376,7 +1938,8 @@ static void query_rpc(hg_handle_t handle) get_query_odscs(server, &in, timeout, &results, req_id); out.odsc_list.raw_odsc = (char *)results; - DEBUG_OUT("Responding with %li result(s).\n", out.odsc_list.size / sizeof(obj_descriptor)); + DEBUG_OUT("Responding with %li result(s).\n", + out.odsc_list.size / sizeof(obj_descriptor)); margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); @@ -2609,7 +2172,7 @@ static void get_rpc(hg_handle_t handle) return; } - suppress_compression = ~(in.flags & DS_NO_COMPRESS); + suppress_compression = in.flags & DS_NO_COMPRESS; obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc.raw_odsc, sizeof(in_odsc)); @@ -2635,7 +2198,7 @@ static void get_rpc(hg_handle_t handle) cbuffer = malloc(size); hret = margo_bulk_create(mid, 1, (void **)&cbuffer, &size, HG_BULK_READ_ONLY, &bulk_handle); - DEBUG_OUT("created bulk handle of size %li\n", size); + DEBUG_OUT("created bulk handle of size %" PRIu64 "\n", size); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: (%s): margo_bulk_create() failure\n", __func__); out.ret = dspaces_ERR_MERCURY; @@ -2650,7 +2213,8 @@ static void get_rpc(hg_handle_t handle) /* things like CPU->GPU transfer don't support compression, but we need * the client to tell us. */ csize = LZ4_compress_default(od->data, cbuffer, size, size); - DEBUG_OUT("compressed result from %li to %i bytes.\n", size, csize); + DEBUG_OUT("compressed result from %" PRIu64 " to %i bytes.\n", size, + csize); if(!csize) { DEBUG_OUT( "compressed result could not fit in dst buffer - longer than " @@ -2844,16 +2408,16 @@ static void ss_rpc(hg_handle_t handle) DEBUG_OUT("received ss_rpc\n"); ss_info_hdr ss_data; - ss_data.num_dims = ds_conf.ndim; + ss_data.num_dims = server->conf.ndim; ss_data.num_space_srv = server->dsg->size_sp; - ss_data.max_versions = ds_conf.max_versions; - ss_data.hash_version = ds_conf.hash_version; - ss_data.default_gdim.ndim = ds_conf.ndim; + ss_data.max_versions = server->conf.max_versions; + ss_data.hash_version = server->conf.hash_version; + ss_data.default_gdim.ndim = server->conf.ndim; - for(int i = 0; i < ds_conf.ndim; i++) { + for(int i = 0; i < server->conf.ndim; i++) { ss_data.ss_domain.lb.c[i] = 0; - ss_data.ss_domain.ub.c[i] = ds_conf.dims.c[i] - 1; - ss_data.default_gdim.sizes.c[i] = ds_conf.dims.c[i]; + ss_data.ss_domain.ub.c[i] = server->conf.dims.c[i] - 1; + ss_data.default_gdim.sizes.c[i] = server->conf.dims.c[i]; } out.ss_buf.size = sizeof(ss_info_hdr); @@ -2886,16 +2450,14 @@ static void kill_rpc(hg_handle_t handle) const struct hg_info *info = margo_get_info(handle); dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - int32_t src, rank, parent, child1, child2; + struct ibcast_state *bcast; + int32_t src, rank; int do_kill = 0; margo_get_input(handle, &src); DEBUG_OUT("Received kill signal from %d.\n", src); rank = server->dsg->rank; - parent = (rank - 1) / 2; - child1 = (rank * 2) + 1; - child2 = child1 + 1; ABT_mutex_lock(server->kill_mutex); DEBUG_OUT("Kill tokens remaining: %d\n", @@ -2914,14 +2476,9 @@ static void kill_rpc(hg_handle_t handle) ABT_mutex_unlock(server->kill_mutex); - if((src == -1 || src > rank) && rank > 0) { - send_kill_rpc(server, parent, &rank); - } - if((child1 != src && child1 < server->dsg->size_sp)) { - send_kill_rpc(server, child1, &rank); - } - if((child2 != src && child2 < server->dsg->size_sp)) { - send_kill_rpc(server, child2, &rank); + bcast = ibcast_rpc_start(server, server->kill_id, src, &rank); + if(bcast) { + ibcast_finish(server, bcast); } margo_free_input(handle, &src); @@ -3055,11 +2612,15 @@ static void do_ops_rpc(hg_handle_t handle) buffer = malloc(res_buf_size); cbuffer = malloc(res_buf_size); if(expr->type == DS_VAL_INT) { + DEBUG_OUT("Executing integer operation on %" PRIu64 " elements\n", + res_buf_size / sizeof(int)); #pragma omp for for(i = 0; i < res_buf_size / sizeof(int); i++) { ((int *)buffer)[i] = ds_op_calc_ival(expr, i, &err); } } else if(expr->type == DS_VAL_REAL) { + DEBUG_OUT("Executing real operation on %" PRIu64 " elements\n", + res_buf_size / sizeof(double)); #pragma omp for for(i = 0; i < res_buf_size / sizeof(double); i++) { ((double *)buffer)[i] = ds_op_calc_rval(expr, i, &err); @@ -3068,7 +2629,6 @@ static void do_ops_rpc(hg_handle_t handle) fprintf(stderr, "ERROR: %s: invalid expression data type.\n", __func__); goto cleanup; } - size = res_buf_size; hret = margo_bulk_create(mid, 1, (void **)&cbuffer, &size, HG_BULK_READ_ONLY, &bulk_handle); @@ -3081,7 +2641,7 @@ static void do_ops_rpc(hg_handle_t handle) csize = LZ4_compress_default(buffer, cbuffer, size, size); - DEBUG_OUT("compressed result from %li to %i bytes.\n", size, csize); + DEBUG_OUT("compressed result from %" PRIu64 " to %i bytes.\n", size, csize); if(!csize) { DEBUG_OUT("compressed result could not fit in dst buffer - longer than " "original! Sending uncompressed.\n"); @@ -3123,7 +2683,7 @@ static PyObject *build_ndarray_from_od(struct obj_data *od) int i; for(i = 0; i < ndim; i++) { - dims[i] = (odsc->bb.ub.c[i] - odsc->bb.lb.c[i]) + 1; + dims[(ndim - i) - 1] = (odsc->bb.ub.c[i] - odsc->bb.lb.c[i]) + 1; } arr = PyArray_NewFromDescr(&PyArray_Type, descr, ndim, dims, NULL, data, 0, @@ -3134,6 +2694,7 @@ static PyObject *build_ndarray_from_od(struct obj_data *od) static void pexec_rpc(hg_handle_t handle) { + PyGILState_STATE gstate; margo_instance_id mid = margo_hg_handle_get_instance(handle); const struct hg_info *info = margo_get_info(handle); dspaces_provider_t server = @@ -3201,7 +2762,8 @@ static void pexec_rpc(hg_handle_t handle) ABT_mutex_lock(server->ls_mutex); num_obj = ls_find_all(server->dsg->ls, &in_odsc, &from_objs); if(num_obj > 0) { - DEBUG_OUT("found %i objects in local storage to populate input\n", num_obj); + DEBUG_OUT("found %i objects in local storage to populate input\n", + num_obj); arg_obj = obj_data_alloc(&in_odsc); od_tab = malloc(num_obj * sizeof(*od_tab)); for(i = 0; i < num_obj; i++) { @@ -3217,7 +2779,7 @@ static void pexec_rpc(hg_handle_t handle) } ABT_mutex_unlock(server->ls_mutex); if(num_obj < 1) { - DEBUG_OUT("could not find input object\n"); + DEBUG_OUT("could not find input object\n"); out.length = 0; out.handle = 0; margo_respond(handle, &out); @@ -3227,14 +2789,15 @@ static void pexec_rpc(hg_handle_t handle) } else if(from_objs) { free(from_objs); } + gstate = PyGILState_Ensure(); array = build_ndarray_from_od(arg_obj); - // Race condition? Protect with mutex? if((pklmod == NULL) && (pklmod = PyImport_ImportModuleNoBlock("dill")) == NULL) { margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); + PyGILState_Release(gstate); return; } arg = PyBytes_FromStringAndSize(fn, rdma_size); @@ -3254,6 +2817,7 @@ static void pexec_rpc(hg_handle_t handle) margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); + PyGILState_Release(gstate); return; } @@ -3273,6 +2837,7 @@ static void pexec_rpc(hg_handle_t handle) margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); + PyGILState_Release(gstate); return; } out.length = rdma_size; @@ -3282,6 +2847,7 @@ static void pexec_rpc(hg_handle_t handle) } out.length = 0; } + PyGILState_Release(gstate); if(out.length > 0) { ABT_cond_create(&cond); @@ -3308,10 +2874,251 @@ static void pexec_rpc(hg_handle_t handle) DEBUG_OUT("done with pexec handling\n"); + gstate = PyGILState_Ensure(); Py_XDECREF(array); + PyGILState_Release(gstate); obj_data_free(arg_obj); } DEFINE_MARGO_RPC_HANDLER(pexec_rpc) + +static void mpexec_rpc(hg_handle_t handle) +{ + PyGILState_STATE gstate; + margo_instance_id mid = margo_hg_handle_get_instance(handle); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); + pexec_in_t in; + pexec_out_t out; + hg_return_t hret; + hg_bulk_t bulk_handle; + obj_descriptor *in_odsc, *in_odscs, odsc; + int num_args; + hg_size_t rdma_size; + void *fn = NULL, *res_data; + struct obj_data *od, **arg_objs, *arg_obj, **od_tab, **from_objs = NULL; + int num_obj; + PyObject **arg_arrays, *fnp, *arg, *args, *pres, *pres_bytes; + static PyObject *pklmod = NULL; + ABT_cond cond; + ABT_mutex mtx; + int i, j; + + hret = margo_get_input(handle, &in); + if(hret != HG_SUCCESS) { + fprintf(stderr, + "DATASPACES: ERROR handling %s: margo_get_input() failed with " + "%d.\n", + __func__, hret); + margo_destroy(handle); + return; + } + in_odscs = (obj_descriptor *)in.odsc.raw_odsc; + num_args = in.odsc.size / sizeof(*in_odscs); + + DEBUG_OUT("received mpexec request with %i args\n", num_args); + rdma_size = in.length; + if(rdma_size > 0) { + DEBUG_OUT("function included, length %" PRIu32 " bytes\n", in.length); + fn = malloc(rdma_size); + hret = margo_bulk_create(mid, 1, (void **)&(fn), &rdma_size, + HG_BULK_WRITE_ONLY, &bulk_handle); + + if(hret != HG_SUCCESS) { + // TODO: communicate failure + fprintf(stderr, "ERROR: (%s): margo_bulk_create failed!\n", + __func__); + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + return; + } + + hret = margo_bulk_transfer(mid, HG_BULK_PULL, info->addr, in.handle, 0, + bulk_handle, 0, rdma_size); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_bulk_transfer failed!\n", + __func__); + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_bulk_free(bulk_handle); + margo_destroy(handle); + return; + } + } + margo_bulk_free(bulk_handle); + + arg_objs = calloc(num_args, sizeof(*arg_objs)); + arg_arrays = calloc(num_args, sizeof(*arg_arrays)); + for(i = 0; i < num_args; i++) { + route_request(server, &in_odscs[i], &(server->dsg->default_gdim)); + + in_odsc = &in_odscs[i]; + DEBUG_OUT("searching for local storage objects to satisfy %s\n", + obj_desc_sprint(in_odsc)); + ABT_mutex_lock(server->ls_mutex); + num_obj = ls_find_all(server->dsg->ls, in_odsc, &from_objs); + if(num_obj > 0) { + // need one source of information - metadata case? + if(in_odsc->size == 0) { + in_odsc->size = from_objs[0]->obj_desc.size; + } + DEBUG_OUT("found %i objects in local storage to populate input\n", + num_obj); + arg_objs[i] = obj_data_alloc(in_odsc); + od_tab = malloc(num_obj * sizeof(*od_tab)); + for(j = 0; j < num_obj; j++) { + // Can we skip the intermediate copy? + odsc = from_objs[j]->obj_desc; + DEBUG_OUT("getting data from %s\n", obj_desc_sprint(&odsc)); + bbox_intersect(&in_odsc->bb, &odsc.bb, &odsc.bb); + od_tab[j] = obj_data_alloc(&odsc); + DEBUG_OUT("overlap = %s\n", + obj_desc_sprint(&od_tab[j]->obj_desc)); + ssd_copy(od_tab[j], from_objs[j]); + ssd_copy(arg_objs[i], od_tab[j]); + obj_data_free(od_tab[j]); + } + free(od_tab); + } + ABT_mutex_unlock(server->ls_mutex); + if(num_obj < 1) { + DEBUG_OUT("could not find input object\n"); + out.length = 0; + out.handle = 0; + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + free(arg_arrays); + for(j = 0; j < i; j++) { + obj_data_free(arg_objs[j]); + } + free(arg_objs); + return; + } else if(from_objs) { + free(from_objs); + } + + arg_arrays[i] = build_ndarray_from_od(arg_objs[i]); + DEBUG_OUT("created ndarray for %s\n", + obj_desc_sprint(&arg_objs[i]->obj_desc)); + } + + gstate = PyGILState_Ensure(); + if((pklmod == NULL) && + (pklmod = PyImport_ImportModuleNoBlock("dill")) == NULL) { + out.length = 0; + out.handle = 0; + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + free(arg_arrays); + free(arg_objs); + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + PyGILState_Release(gstate); + return; + } + arg = PyBytes_FromStringAndSize(fn, rdma_size); + fnp = PyObject_CallMethodObjArgs(pklmod, PyUnicode_FromString("loads"), arg, + NULL); + Py_XDECREF(arg); + + args = PyTuple_New(num_args); + for(i = 0; i < num_args; i++) { + PyTuple_SetItem(args, i, arg_arrays[i]); + } + if(fnp && PyCallable_Check(fnp)) { + pres = PyObject_CallObject(fnp, args); + } else { + if(!fnp) { + PyErr_Print(); + } + fprintf(stderr, + "ERROR: (%s): provided function could either not be loaded, or " + "is not callable.\n", + __func__); + out.length = 0; + out.handle = 0; + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + free(arg_arrays); + free(arg_objs); + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + PyGILState_Release(gstate); + return; + } + + if(pres && (pres != Py_None)) { + pres_bytes = PyObject_CallMethodObjArgs( + pklmod, PyUnicode_FromString("dumps"), pres, NULL); + Py_XDECREF(pres); + res_data = PyBytes_AsString(pres_bytes); + rdma_size = PyBytes_Size(pres_bytes) + 1; + hret = margo_bulk_create(mid, 1, (void **)&res_data, &rdma_size, + HG_BULK_READ_ONLY, &out.handle); + if(hret != HG_SUCCESS) { + fprintf(stderr, "ERROR: (%s): margo_bulk_create failed with %d.\n", + __func__, hret); + out.length = 0; + out.handle = 0; + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + PyGILState_Release(gstate); + return; + } + out.length = rdma_size; + } else { + if(!pres) { + PyErr_Print(); + } + out.length = 0; + } + PyGILState_Release(gstate); + + if(out.length > 0) { + ABT_cond_create(&cond); + ABT_mutex_create(&mtx); + out.condp = (uint64_t)(&cond); + out.mtxp = (uint64_t)(&mtx); + DEBUG_OUT("sending out.condp = %" PRIu64 " and out.mtxp = %" PRIu64 + "\n", + out.condp, out.mtxp); + ABT_mutex_lock(mtx); + margo_respond(handle, &out); + ABT_cond_wait(cond, mtx); + DEBUG_OUT("signaled on condition\n"); + ABT_mutex_unlock(mtx); + ABT_mutex_free(&mtx); + ABT_cond_free(&cond); + } else { + out.handle = 0; + margo_respond(handle, &out); + } + + margo_free_input(handle, &in); + margo_destroy(handle); + + DEBUG_OUT("done with pexec handling\n"); + + gstate = PyGILState_Ensure(); + Py_XDECREF(args); + PyGILState_Release(gstate); + free(arg_arrays); + if(num_args) { + for(i = 0; i < num_args; i++) { + obj_data_free(arg_objs[i]); + } + free(arg_objs); + } +} +DEFINE_MARGO_RPC_HANDLER(mpexec_rpc) + #endif // DSPACES_HAVE_PYTHON static void cond_rpc(hg_handle_t handle) @@ -3338,59 +3145,27 @@ static void cond_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(cond_rpc) -static void send_tgt_rpc(dspaces_provider_t server, hg_id_t rpc_id, int target, - void *in, hg_addr_t *addr, hg_handle_t *h, - margo_request *req) -{ - margo_addr_lookup(server->mid, server->server_address[target], addr); - margo_create(server->mid, *addr, rpc_id, h); - margo_iforward(*h, in, req); -} - static void get_vars_rpc(hg_handle_t handle) { margo_instance_id mid = margo_hg_handle_get_instance(handle); const struct hg_info *info = margo_get_info(handle); dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - int32_t src, rank, parent, child1, child2; - int sent_rpc[3] = {0}; - hg_addr_t addr[3]; - hg_handle_t hndl[3]; - margo_request req[3]; - name_list_t out[3]; - name_list_t rout; + struct ibcast_state *bcast; + int32_t src, rank; + name_list_t out[3] = {0}; + name_list_t rout = {0}; ds_str_hash *results = ds_str_hash_init(); + int present; int num_vars; - char **names; + char **names = NULL; int i, j; margo_get_input(handle, &src); DEBUG_OUT("Received request for all variable names from %d\n", src); rank = server->dsg->rank; - parent = (rank - 1) / 2; - child1 = (rank * 2) + 1; - child2 = child1 + 1; - - if((src == -1 || src > rank) && rank > 0) { - DEBUG_OUT("querying parent %d\n", parent); - send_tgt_rpc(server, server->get_vars_id, parent, &rank, &addr[0], - &hndl[0], &req[0]); - sent_rpc[0] = 1; - } - if((child1 != src && child1 < server->dsg->size_sp)) { - DEBUG_OUT("querying child %d\n", child1); - send_tgt_rpc(server, server->get_vars_id, child1, &rank, &addr[1], - &hndl[1], &req[1]); - sent_rpc[1] = 1; - } - if((child2 != src && child2 < server->dsg->size_sp)) { - DEBUG_OUT("querying child %d\n", child2); - send_tgt_rpc(server, server->get_vars_id, child2, &rank, &addr[2], - &hndl[2], &req[2]); - sent_rpc[2] = 1; - } + bcast = ibcast_rpc_start(server, server->get_vars_id, src, &rank); ABT_mutex_lock(server->ls_mutex); num_vars = ls_get_var_names(server->dsg->ls, &names); @@ -3402,23 +3177,24 @@ static void get_vars_rpc(hg_handle_t handle) ds_str_hash_add(results, names[i]); free(names[i]); } - free(names); + if(names) { + free(names); + } - for(i = 0; i < 3; i++) { - if(sent_rpc[i]) { - margo_wait(req[i]); - margo_get_output(hndl[i], &out[i]); - for(j = 0; j < out[i].count; j++) { - ds_str_hash_add(results, out[i].names[j]); + if(bcast) { + for(i = 0; i < 3; i++) { + ibcast_get_output(bcast, i, &out[i], &present); + if(present) { + for(j = 0; j < out[i].count; j++) { + ds_str_hash_add(results, out[i].names[j]); + } } - margo_free_output(hndl[i], &out); - margo_addr_free(server->mid, addr[i]); - margo_destroy(hndl[i]); } + ibcast_finish(server, bcast); } rout.count = ds_str_hash_get_all(results, &rout.names); - DEBUG_OUT("returning %zi variable names\n", rout.count); + DEBUG_OUT("returning %" PRIu64 " variable names\n", rout.count); margo_respond(handle, &rout); for(i = 0; i < rout.count; i++) { free(rout.names[i]); @@ -3436,16 +3212,14 @@ static void get_var_objs_rpc(hg_handle_t handle) const struct hg_info *info = margo_get_info(handle); dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - get_var_objs_in_t in, tgt_in; - int32_t src, rank, parent, child1, child2; - int sent_rpc[3] = {0}; - hg_addr_t addr[3]; - hg_handle_t hndl[3]; - margo_request req[3]; + get_var_objs_in_t in, bcast_in; + struct ibcast_state *bcast; + int32_t src, rank; odsc_hdr out[3]; obj_descriptor **odsc_tab; int i; int num_odscs; + int present; odsc_hdr rout; hg_return_t hret; @@ -3456,31 +3230,11 @@ static void get_var_objs_rpc(hg_handle_t handle) src); rank = server->dsg->rank; - parent = (rank - 1) / 2; - child1 = (rank * 2) + 1; - child2 = child1 + 1; - tgt_in.src = rank; - tgt_in.var_name = in.var_name; + bcast_in.src = rank; + bcast_in.var_name = in.var_name; - if((src == -1 || src > rank) && rank > 0) { - DEBUG_OUT("querying parent %d\n", parent); - send_tgt_rpc(server, server->get_var_objs_id, parent, &tgt_in, &addr[0], - &hndl[0], &req[0]); - sent_rpc[0] = 1; - } - if((child1 != src && child1 < server->dsg->size_sp)) { - DEBUG_OUT("querying child %d\n", child1); - send_tgt_rpc(server, server->get_var_objs_id, child1, &tgt_in, &addr[1], - &hndl[1], &req[1]); - sent_rpc[1] = 1; - } - if((child2 != src && child2 < server->dsg->size_sp)) { - DEBUG_OUT("querying child %d\n", child2); - send_tgt_rpc(server, server->get_var_objs_id, child2, &tgt_in, &addr[2], - &hndl[2], &req[2]); - sent_rpc[2] = 1; - } + bcast = ibcast_rpc_start(server, server->get_var_objs_id, src, &bcast_in); ABT_mutex_lock(server->ls_mutex); num_odscs = ls_find_all_no_version(server->dsg->ls, in.var_name, &odsc_tab); @@ -3498,17 +3252,16 @@ static void get_var_objs_rpc(hg_handle_t handle) DEBUG_OUT("found %d object descriptors locally.\n", num_odscs); - for(i = 0; i < 3; i++) { - if(sent_rpc[i]) { - margo_wait(req[i]); - margo_get_output(hndl[i], &out[i]); - rout.raw_odsc = realloc(rout.raw_odsc, rout.size + out[i].size); - memcpy(rout.raw_odsc + rout.size, out[i].raw_odsc, out[i].size); - rout.size += out[i].size; - margo_free_output(hndl[i], &out); - margo_addr_free(server->mid, addr[i]); - margo_destroy(hndl[i]); + if(bcast) { + for(i = 0; i < 3; i++) { + ibcast_get_output(bcast, i, &out[i], &present); + if(present) { + rout.raw_odsc = realloc(rout.raw_odsc, rout.size + out[i].size); + memcpy(rout.raw_odsc + rout.size, out[i].raw_odsc, out[i].size); + rout.size += out[i].size; + } } + ibcast_finish(server, bcast); } DEBUG_OUT("returning %zi objects.\n", rout.size / sizeof(obj_descriptor)); @@ -3522,18 +3275,151 @@ static void get_var_objs_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(get_var_objs_rpc) -void dspaces_server_fini(dspaces_provider_t server) +static int dspaces_init_registry(dspaces_provider_t server) { + struct dspaces_module *mod; + struct dspaces_module_args arg; + struct dspaces_module_ret *res = NULL; int err; + server->local_reg_id = 0; + + mod = dspaces_mod_by_name(&server->mods, "ds_reg"); + + if(!mod) { + fprintf(stderr, "missing built-in ds_reg module.\n"); + return (-1); + } + + build_module_arg_from_rank(server->rank, &arg); + res = + dspaces_module_exec(mod, "bootstrap_id", &arg, 1, DSPACES_MOD_RET_INT); + if(res && res->type == DSPACES_MOD_RET_ERR) { + err = res->err; + free(res); + return (err); + } + if(res) { + server->local_reg_id = res->ival; + } + + DEBUG_OUT("local registry id starting at %i\n", server->local_reg_id); + + return (0); +} + +static int route_registration(dspaces_provider_t server, reg_in_t *reg) +{ + struct dspaces_module *mod, *reg_mod; + int nargs; + struct dspaces_module_ret *res = NULL; + struct dspaces_module_args *args; + int err; + + DEBUG_OUT("routing registration request.\n"); + + mod = dspaces_mod_by_name(&server->mods, reg->type); + if(!mod) { + DEBUG_OUT("could not find module for type '%s'.\n", reg->type); + return (DS_MOD_ENOMOD); + } + + // ds_reg module doesn't have access to the map between dspaces module name + // and Python module name, so translate for it. + free(reg->type); + reg->type = strdup(mod->file); + + reg_mod = dspaces_mod_by_name(&server->mods, "ds_reg"); + if(!reg_mod) { + return (DS_MOD_EFAULT); + } + nargs = build_module_args_from_reg(reg, &args); + res = dspaces_module_exec(reg_mod, "register", args, nargs, + DSPACES_MOD_RET_INT); + if(!res) { + return (DS_MOD_EFAULT); + } + if(res->type == DSPACES_MOD_RET_ERR) { + err = res->err; + free(res); + return (err); + } + if(res->ival != reg->id) { + DEBUG_OUT("updating registration id to %li.\n", res->ival); + reg->id = res->ival; + } + + return (0); +} + +static void reg_rpc(hg_handle_t handle) +{ + margo_instance_id mid = margo_hg_handle_get_instance(handle); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); + hg_return_t hret; + reg_in_t in; + uint64_t out; + struct ibcast_state *bcast; + int i, err; + + mid = margo_hg_handle_get_instance(handle); + info = margo_get_info(handle); + server = (dspaces_provider_t)margo_registered_data(mid, info->id); + hret = margo_get_input(handle, &in); + if(hret != HG_SUCCESS) { + fprintf(stderr, + "DATASPACES: ERROR handling %s: margo_get_input() failed with " + "%d.\n", + __func__, hret); + margo_destroy(handle); + return; + } + + DEBUG_OUT("received registration '%s' of type '%s' with %zi bytes of " + "registration data.\n", + in.name, in.type, strlen(in.reg_data)); + + if(in.src == -1) { + in.id = __sync_fetch_and_add(&server->local_reg_id, 1); + in.id += (uint64_t)server->dsg->rank << 40; + } + bcast = ibcast_rpc_start(server, server->reg_id, in.src, &in); + if(bcast) { + for(i = 0; i < 3; i++) { + ibcast_get_output(bcast, i, &out, NULL); + } + ibcast_finish(server, bcast); + } + + err = route_registration(server, &in); + if(err != 0) { + out = err; + } else { + out = in.id; + } + + margo_free_input(handle, &in); + margo_respond(handle, &out); + margo_destroy(handle); +} +DEFINE_MARGO_RPC_HANDLER(reg_rpc); + +void dspaces_server_fini(dspaces_provider_t server) +{ + int err = 0; + DEBUG_OUT("waiting for finalize to occur\n"); margo_wait_for_finalize(server->mid); #ifdef DSPACES_HAVE_PYTHON + PyEval_RestoreThread(server->main_state); err = Py_FinalizeEx(); #endif // DSPACES_HAVE_PYTHON if(err < 0) { fprintf(stderr, "ERROR: Python finalize failed with %d\n", err); } + DEBUG_OUT("finalize complete\n"); free(server); } diff --git a/src/ss_data.c b/src/ss_data.c index 2ad070b1..c0e9b104 100644 --- a/src/ss_data.c +++ b/src/ss_data.c @@ -372,6 +372,45 @@ static int dht_construct_hash(struct dht *dht, struct sspace *ssd) return err; } +static struct sspace *ssd_alloc_v0(const struct bbox *bb_domain, + int max_versions) +{ + struct sspace *ssd; + int err = -ENOMEM; + + ssd = malloc(sizeof(*ssd)); + if(!ssd) + goto err_out; + memset(ssd, 0, sizeof(*ssd)); + + ssd->dht = dht_alloc(ssd, bb_domain, 1, max_versions); + if(!ssd->dht) { + free(ssd); + goto err_out; + } + + ssd->dht->ent_tab[0]->rank = 0; + + ssd->hash_version = ssd_hash_version_v0; + return ssd; +err_out: + fprintf(stderr, "'%s()': failed with %d\n", __func__, err); + return NULL; +} + +static int ssd_hash_v0(struct sspace *ss, const struct bbox *bb, + struct dht_entry *de_tab[]) +{ + de_tab[0] = ss->dht->ent_tab[0]; + return 1; +} + +static void ssd_free_v0(struct sspace *ssd) +{ + dht_free(ssd->dht); + free(ssd); +} + static struct sspace *ssd_alloc_v1(const struct bbox *bb_domain, int num_nodes, int max_versions) { @@ -752,9 +791,10 @@ char *obj_desc_sprint(obj_descriptor *odsc) "\t.version = %d,\n" "\t.size = %i,\n" "\t.tag = %i,\n" + "\t.type = %i,\n" "\t.bb = ", odsc->name, odsc->owner, odsc->version, odsc->size, - odsc->tag); + odsc->tag, odsc->type); str = str_append_const(str_append(str, bbox_sprint(&odsc->bb)), "}\n"); return str; @@ -766,6 +806,8 @@ int ssd_copy(struct obj_data *to_obj, struct obj_data *from_obj) struct bbox bbcom; int copied_elems = 0; + to_obj->obj_desc.tag = from_obj->obj_desc.tag; + to_obj->obj_desc.type = from_obj->obj_desc.type; bbox_intersect(&to_obj->obj_desc.bb, &from_obj->obj_desc.bb, &bbcom); matrix_init(&from_mat, from_obj->obj_desc.st, &from_obj->obj_desc.bb, @@ -932,9 +974,6 @@ void ls_add_obj(ss_storage *ls, struct obj_data *od) struct list_head *bin; struct obj_data *od_existing; - // ABT_rwlock_create(&od->lock); - // ABT_rwlock_wrlock(&od->lock); - // duplicate is noop ds_str_hash_add(ls->var_dict, od->obj_desc.name); @@ -954,7 +993,6 @@ void ls_add_obj(ss_storage *ls, struct obj_data *od) /* NOTE: new object comes first in the list. */ list_add(&od->obj_entry, bin); ls->num_obj++; - // ABT_rwlock_unlock(&od->lock); } struct obj_data *ls_lookup(ss_storage *ls, char *name) @@ -1045,24 +1083,24 @@ int ls_find_all(ss_storage *ls, obj_descriptor *odsc, struct obj_data ***ods) struct bbox isect; num_elem = bbox_volume(&odsc->bb); - *ods = malloc(sizeof(**ods) * ls->num_obj); + *ods = malloc(sizeof(**ods) * ls->num_obj); index = odsc->version % ls->size_hash; list = &ls->obj_hash[index]; list_for_each_entry(od, list, struct obj_data, obj_entry) { - if(obj_desc_equals_intersect(odsc, &od->obj_desc)) { + if(obj_desc_equals_intersect(odsc, &od->obj_desc)) { (*ods)[n++] = od; bbox_intersect(&odsc->bb, &od->obj_desc.bb, &isect); num_elem -= bbox_volume(&isect); - } + } } if(num_elem != 0) { free(*ods); - return(-1); + return (-1); } - - return(n); + + return (n); } /* @@ -1318,6 +1356,11 @@ int ssd_choose_hash(const struct bbox *bb_domain) uint64_t x; int i; + if(bb_domain->num_dims == 1 && bb_domain->ub.c[0] == 0) { + // flag for no hashing + return (ssd_hash_version_v0); + } + for(i = 0; i < bb_domain->num_dims; i++) { x = bb_domain->ub.c[i]; if(x & x + 1) { @@ -1357,6 +1400,9 @@ struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, } switch(hash_version) { + case ssd_hash_version_v0: + ss = ssd_alloc_v0(bb_domain, max_versions); + break; case ssd_hash_version_v1: ss = ssd_alloc_v1(bb_domain, num_nodes, max_versions); break; @@ -1381,6 +1427,9 @@ struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, void ssd_free(struct sspace *ss) { switch(ss->hash_version) { + case ssd_hash_version_v0: + ssd_free_v0(ss); + break; case ssd_hash_version_v1: ssd_free_v1(ss); break; @@ -1394,6 +1443,11 @@ void ssd_free(struct sspace *ss) } } +long ssh_hash_elem_count_v0(struct sspace *ss, const struct bbox *bb) +{ + return (bbox_volume(bb)); +} + long ssh_hash_elem_count_v1(struct sspace *ss, const struct bbox *bb) { struct intv *i_tab, *i_self; @@ -1453,6 +1507,9 @@ long ssh_hash_elem_count(struct sspace *ss, const struct bbox *bb) long ret; switch(ss->hash_version) { + case ssd_hash_version_v0: + ret = ssh_hash_elem_count_v0(ss, bb); + break; case ssd_hash_version_v1: ret = ssh_hash_elem_count_v1(ss, bb); break; @@ -1487,6 +1544,9 @@ int ssd_hash(struct sspace *ss, const struct bbox *bb, int ret; switch(ss->hash_version) { + case ssd_hash_version_v0: + ret = ssd_hash_v0(ss, bb, de_tab); + break; case ssd_hash_version_v1: ret = ssd_hash_v1(ss, bb, de_tab); break; @@ -1725,7 +1785,7 @@ int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, long num_elem; struct obj_desc_list *odscl; struct bbox isect; - int sub = timeout != 0 && de == de->ss->ent_self; + int sub = (timeout == -1 && de == de->ss->ent_self); n = q_odsc->version % de->odsc_size; num_elem = ssh_hash_elem_count(de->ss, &q_odsc->bb); diff --git a/tests/dspaces_ls.c b/tests/dspaces_ls.c index f6d1bbc6..85fd791d 100644 --- a/tests/dspaces_ls.c +++ b/tests/dspaces_ls.c @@ -29,6 +29,8 @@ int ls_all(dspaces_client_t dsp) if(count) { free(names); } + + return(0); } int ls_one(dspaces_client_t dsp, const char *name) @@ -59,6 +61,8 @@ int ls_one(dspaces_client_t dsp, const char *name) if(count) { free(found_objs); } + + return(0); } int main(int argc, char **argv) diff --git a/tests/test_calc.c b/tests/test_calc.c index 1be84ee9..a844180b 100644 --- a/tests/test_calc.c +++ b/tests/test_calc.c @@ -1,17 +1,18 @@ /* - * Copyright (c) 2022, Scientific Computing & Imagine Institute, University of Utah + * Copyright (c) 2022, Scientific Computing & Imagine Institute, University of + * Utah * * See COPYRIGHT in top-level directory. */ #include -#include #include +#include -#include -#include -#include +#include +#include +#include int main(int argc, char **argv) { @@ -53,13 +54,16 @@ int main(int argc, char **argv) one_minus_a_plus_b = dspaces_op_new_sub(const_1, a_plus_b); dspaces_op_calc(client, one_minus_a_plus_b, (void **)&result); for(i = 0; i < gdim[0]; i++) { - for(j = 0; j < gdim[1]; j++) { - if(result[(i * gdim[1]) + j] != 1 - (a[(i * gdim[1]) + j] + b[(i * gdim[1]) + j])) { + for(j = 0; j < gdim[1]; j++) { + if(result[(i * gdim[1]) + j] != + 1 - (a[(i * gdim[1]) + j] + b[(i * gdim[1]) + j])) { fprintf(stderr, "Bad value at (%i, %i)\n", i, j); - fprintf(stderr, " Expected %lf, but got %lf\n", 1 - (a[(i * gdim[1]) + j] + b[(i * gdim[1]) + j]), result[(i * gdim[1]) + j]); - return(-1); + fprintf(stderr, " Expected %lf, but got %lf\n", + 1 - (a[(i * gdim[1]) + j] + b[(i * gdim[1]) + j]), + result[(i * gdim[1]) + j]); + return (-1); } - } + } } fprintf(stderr, "result[100] = %lf\n", result[100]); diff --git a/tests/test_get_run.c b/tests/test_get_run.c index beac0449..59460004 100644 --- a/tests/test_get_run.c +++ b/tests/test_get_run.c @@ -109,6 +109,7 @@ static int couple_read_nd(dspaces_client_t client, unsigned int ts, unsigned int meta_len; int next_ver; uint64_t lb[10] = {0}, ub[10] = {0}; + struct dspaces_req in_req = {0}, out_req = {0}; for(i = 0; i < dims; i++) { lb[i] = off[i]; ub[i] = off[i] + sp[i] - 1; @@ -166,8 +167,16 @@ static int couple_read_nd(dspaces_client_t client, unsigned int ts, for(i = 0; i < num_vars; i++) { sprintf(var_name, "mnd_%d", i); if(allocate) { - err = dspaces_aget(client, var_name, ts, dims, lb, ub, - (void **)&data_tab[i], NULL, -1); + // err = dspaces_aget(client, var_name, ts, dims, lb, ub, + // (void **)&data_tab[i], NULL, -1); + in_req.var_name = var_name; + in_req.ver = ts; + in_req.ndim = dims; + in_req.lb = lb; + in_req.ub = ub; + err = dspaces_get_req(client, &in_req, &out_req, -1); + data_tab[i] = out_req.buf; + } else { err = dspaces_get(client, var_name, ts, elem_size, dims, lb, ub, data_tab[i], -1); diff --git a/tests/test_script.sh.in b/tests/test_script.sh.in index 938ac3c4..2fee72c6 100755 --- a/tests/test_script.sh.in +++ b/tests/test_script.sh.in @@ -147,14 +147,12 @@ elif [ $1 -eq 12 ] ; then wait $serverproc || exit 1 wait $wproc elif [ $1 -eq 13 ] ; then - @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 dspaces_server sockets & - @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 test_writer 1 1 64 5 -s 8 -m $storage_mode -t & - wproc=$! + @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 2 dspaces_server sockets & + @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 test_writer 1 1 64 5 -s 8 -m $storage_mode -t if [ $? != 0 ] ; then exit 1 fi dspaces_ls @MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 terminator wait $serverproc || exit 1 - wait $wproc fi diff --git a/tests/test_writer_server.c b/tests/test_writer_server.c index f28e9886..b466cd59 100644 --- a/tests/test_writer_server.c +++ b/tests/test_writer_server.c @@ -11,6 +11,7 @@ #include #include #include +#include extern int test_put_run(int dims, int *npdim, uint64_t *spdim, int timestep, size_t elem_size, int num_vars, int local_mode, @@ -174,6 +175,8 @@ int main(int argc, char **argv) if(ret != 0) return ret; + sleep(2); + test_put_run(dims, np, sp, timestep, elem_size, num_vars, local_mode, terminate, nonblock, gcomm);