From 4420a0cf4a657710d09b79a6f058b8f506a2a45c Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Wed, 24 Apr 2024 12:47:08 -0400 Subject: [PATCH 01/10] Add Python bindings for dspaces_get_vars() dspaces_get_var_objs() --- bindings/python/dspaces.py | 25 +++++ bindings/python/dspaces_wrapper.c | 175 ++++++++++++++++++++++-------- bindings/python/dspaces_wrapper.h | 26 +++-- 3 files changed, 172 insertions(+), 54 deletions(-) diff --git a/bindings/python/dspaces.py b/bindings/python/dspaces.py index 4d28754e..d7119ccc 100644 --- a/bindings/python/dspaces.py +++ b/bindings/python/dspaces.py @@ -1,7 +1,15 @@ +from dataclasses import dataclass from dspaces.dspaces_wrapper import * import numpy as np import dill as pickle +@dataclass +class DSObject: + name: str + version: int + lb: tuple[int, ...] + ub: tuple[int, ...] + class DSServer: def __init__(self, conn = "sockets", comm = None, conf = "dataspaces.conf"): from mpi4py import MPI @@ -64,6 +72,23 @@ def Exec(self, name, version, lb=None, ub=None, fn=None): def DefineGDim(self, name, gdim): wrapper_dspaces_define_gdim(self.client, (self.nspace + name).encode('ascii'), gdim) + def GetVars(self): + return wrapper_dspaces_get_vars(self.client) + + def GetObjVars(self, var_name): + ret_objs = [] + wrapper_results = wrapper_dspaces_get_var_objs(self.client, var_name.encode('ascii')) + for obj in wrapper_results: + ret_objs.append( + DSObject( + name = obj['name'], + version = obj['version'], + lb = obj['lb'], + ub = obj['ub'] + ) + ) + return(ret_objs) + def _get_expr(obj, client): if isinstance(obj, DSExpr): return(obj) diff --git a/bindings/python/dspaces_wrapper.c b/bindings/python/dspaces_wrapper.c index f93b15cf..a77d2632 100644 --- a/bindings/python/dspaces_wrapper.c +++ b/bindings/python/dspaces_wrapper.c @@ -5,9 +5,9 @@ #include #include -#include #include #include +#include #include @@ -46,7 +46,7 @@ PyObject *wrapper_dspaces_init_mpi(PyObject *commpy) comm_p = PyMPIComm_Get(commpy); if(!comm_p) { - return(NULL); + return (NULL); } clientp = malloc(sizeof(*clientp)); @@ -62,7 +62,8 @@ PyObject *wrapper_dspaces_init_mpi(PyObject *commpy) return (client); } -PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, int rank) +PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, + int rank) { dspaces_client_t *clientp; char err_str[100]; @@ -85,7 +86,8 @@ PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, int return (client); } -PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, PyObject *commpy) +PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, + PyObject *commpy) { MPI_Comm *comm_p = NULL; dspaces_client_t *clientp; @@ -97,7 +99,7 @@ PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, comm_p = PyMPIComm_Get(commpy); if(!comm_p) { - return(NULL); + return (NULL); } clientp = malloc(sizeof(*clientp)); @@ -106,16 +108,15 @@ PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, sprintf(err_str, "dspaces_init_wan_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); return NULL; - } - + } PyObject *client = PyLong_FromVoidPtr((void *)clientp); - return(client); + return (client); } PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, - const char *conf) + const char *conf) { MPI_Comm *comm_p = NULL; dspaces_provider_t *serverp; @@ -127,7 +128,7 @@ PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, comm_p = PyMPIComm_Get(commpy); if(!comm_p) { - return(NULL); + return (NULL); } serverp = malloc(sizeof(*serverp)); @@ -216,17 +217,18 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, ub[i] = PyLong_AsLong(item); } - Py_BEGIN_ALLOW_THREADS - dspaces_aget(*clientp, name, version, ndim, lb, ub, &data, &tag, timeout); + Py_BEGIN_ALLOW_THREADS dspaces_aget(*clientp, name, version, ndim, lb, ub, + &data, &tag, timeout); Py_END_ALLOW_THREADS - if(data == NULL) { + if(data == NULL) + { Py_INCREF(Py_None); - return(Py_None); + return (Py_None); } if(dtype == Py_None) { - descr = PyArray_DescrNewFromType(tag); + descr = PyArray_DescrNewFromType(tag); } else { descr = PyArray_DescrNew((PyArray_Descr *)dtype); } @@ -242,8 +244,8 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, } PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, - int version, PyObject *lbt, PyObject *ubt, - PyObject *fn, const char *fn_name) + int version, PyObject *lbt, PyObject *ubt, + PyObject *fn, const char *fn_name) { dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); PyObject *item; @@ -255,13 +257,14 @@ PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, int i; if(lbt == Py_None || ubt == Py_None) { - if (lbt != ubt) { - PyErr_SetString(PyExc_TypeError, "both lb and ub must be set or neither"); - return(NULL); + if(lbt != ubt) { + PyErr_SetString(PyExc_TypeError, + "both lb and ub must be set or neither"); + return (NULL); } } else if(PyTuple_GET_SIZE(lbt) != PyTuple_GET_SIZE(ubt)) { PyErr_SetString(PyExc_TypeError, "lb and ub must have the same lenght"); - return(NULL); + return (NULL); } else { ndim = PyTuple_GET_SIZE(lbt); } @@ -276,26 +279,29 @@ PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, } if(!PyBytes_Check(fn)) { - PyErr_SetString(PyExc_TypeError, "fn must be serialized as a a byte string"); - return(NULL); + PyErr_SetString(PyExc_TypeError, + "fn must be serialized as a a byte string"); + return (NULL); } - dspaces_pexec(*clientp, name, version, ndim, lb, ub, PyBytes_AsString(fn), PyBytes_Size(fn)+1, fn_name, &data, &data_size); + dspaces_pexec(*clientp, name, version, ndim, lb, ub, PyBytes_AsString(fn), + PyBytes_Size(fn) + 1, fn_name, &data, &data_size); free(lb); free(ub); - if(data_size > 0) { + if(data_size > 0) { result = PyBytes_FromStringAndSize(data, data_size); } else { Py_INCREF(Py_None); result = Py_None; } - return(result); + return (result); } -void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, PyObject *gdimt) +void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, + PyObject *gdimt) { dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); int ndim = PyTuple_GET_SIZE(gdimt); @@ -305,12 +311,81 @@ void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, PyObject for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(gdimt, i); - gdim[i] = PyLong_AsLong(item); + gdim[i] = PyLong_AsLong(item); } dspaces_define_gdim(*clientp, name, ndim, gdim); } +PyObject *wrapper_dspaces_get_vars(PyObject *clientppy) +{ + dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); + int num_vars; + char **var_names = NULL; + PyObject *var_list; + PyObject *name; + int i; + + num_vars = dspaces_get_var_names(*clientp, &var_names); + if(num_vars < 0) { + return (NULL); + } + + var_list = PyList_New(0); + for(i = 0; i < num_vars; i++) { + name = PyUnicode_DecodeASCII(var_names[i], strlen(var_names[i]), NULL); + if(name) { + PyList_Append(var_list, name); + } + free(var_names[i]); + } + + if(num_vars) { + free(var_names); + } + + return (var_list); +} + +PyObject *wrapper_dspaces_get_var_objs(PyObject *clientppy, const char *name) +{ + dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); + int num_obj; + dspaces_obj_t *objs, *obj; + PyObject *obj_list; + PyObject *pobj, *lbt, *ubt; + int i, j; + + num_obj = dspaces_get_var_objs(*clientp, name, &objs); + if(num_obj < 0) { + return (NULL); + } + + obj_list = PyList_New(0); + for(i = 0; i < num_obj; i++) { + obj = &objs[i]; + pobj = PyDict_New(); + PyDict_SetItemString( + pobj, "name", + PyUnicode_DecodeASCII(obj->name, strlen(obj->name), NULL)); + PyDict_SetItemString(pobj, "version", PyLong_FromLong(obj->version)); + lbt = PyTuple_New(obj->ndim); + ubt = PyTuple_New(obj->ndim); + for(j = 0; j < obj->ndim; j++) { + PyTuple_SetItem(lbt, j, PyLong_FromLong(obj->lb[j])); + PyTuple_SetItem(ubt, j, PyLong_FromLong(obj->ub[j])); + } + PyDict_SetItemString(pobj, "lb", lbt); + PyDict_SetItemString(pobj, "ub", ubt); + PyList_Append(obj_list, pobj); + free(obj->name); + free(obj->lb); + free(obj->ub); + } + + return (obj_list); +} + PyObject *wrapper_dspaces_ops_new_iconst(long val) { ds_expr_t *exprp; @@ -321,7 +396,7 @@ PyObject *wrapper_dspaces_ops_new_iconst(long val) PyObject *expr = PyLong_FromVoidPtr((void *)exprp); - return(expr); + return (expr); } PyObject *wrapper_dspaces_ops_new_rconst(double val) @@ -334,10 +409,12 @@ PyObject *wrapper_dspaces_ops_new_rconst(double val) PyObject *expr = PyLong_FromVoidPtr((void *)exprp); - return(expr); + return (expr); } -PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, int version, PyObject *lbt, PyObject *ubt, PyObject *dtype) +PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, + int version, PyObject *lbt, PyObject *ubt, + PyObject *dtype) { dspaces_client_t *clientp = PyLong_AsVoidPtr(clientppy); ds_expr_t *exprp; @@ -359,7 +436,7 @@ PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, int val_type = DS_VAL_INT; } else { PyErr_SetString(PyExc_TypeError, "type must be int or float"); - return(NULL); + return (NULL); } for(i = 0; i < ndim; i++) { @@ -370,10 +447,11 @@ PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, int } exprp = malloc(sizeof(*exprp)); - *exprp = dspaces_op_new_obj(*clientp, name, version, val_type, ndim, lb, ub); + *exprp = + dspaces_op_new_obj(*clientp, name, version, val_type, ndim, lb, ub); expr = PyLong_FromVoidPtr((void *)exprp); - return(expr); + return (expr); } PyObject *wrapper_dspaces_op_new_add(PyObject *exprppy1, PyObject *exprppy2) @@ -388,7 +466,7 @@ PyObject *wrapper_dspaces_op_new_add(PyObject *exprppy1, PyObject *exprppy2) *resp = dspaces_op_new_add(*exprp1, *exprp2); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_op_new_sub(PyObject *exprppy1, PyObject *exprppy2) @@ -403,7 +481,7 @@ PyObject *wrapper_dspaces_op_new_sub(PyObject *exprppy1, PyObject *exprppy2) *resp = dspaces_op_new_sub(*exprp1, *exprp2); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_op_new_mult(PyObject *exprppy1, PyObject *exprppy2) @@ -418,7 +496,7 @@ PyObject *wrapper_dspaces_op_new_mult(PyObject *exprppy1, PyObject *exprppy2) *resp = dspaces_op_new_mult(*exprp1, *exprp2); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_op_new_div(PyObject *exprppy1, PyObject *exprppy2) @@ -433,7 +511,7 @@ PyObject *wrapper_dspaces_op_new_div(PyObject *exprppy1, PyObject *exprppy2) *resp = dspaces_op_new_div(*exprp1, *exprp2); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_op_new_pow(PyObject *exprppy1, PyObject *exprppy2) @@ -448,7 +526,7 @@ PyObject *wrapper_dspaces_op_new_pow(PyObject *exprppy1, PyObject *exprppy2) *resp = dspaces_op_new_pow(*exprp1, *exprp2); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_op_new_arctan(PyObject *exprppy1) @@ -462,7 +540,7 @@ PyObject *wrapper_dspaces_op_new_arctan(PyObject *exprppy1) *resp = dspaces_op_new_arctan(*exprp1); res = PyLong_FromVoidPtr((void *)resp); - return(res); + return (res); } PyObject *wrapper_dspaces_ops_calc(PyObject *clientppy, PyObject *exprppy) @@ -487,14 +565,16 @@ PyObject *wrapper_dspaces_ops_calc(PyObject *clientppy, PyObject *exprppy) if(etype == DS_VAL_INT) { int_res = *(long *)result_buf; free(result_buf); - return(PyLong_FromLong(int_res)); + return (PyLong_FromLong(int_res)); } else if(etype == DS_VAL_REAL) { real_res = *(double *)result_buf; free(result_buf); - return(PyFloat_FromDouble(real_res)); + return (PyFloat_FromDouble(real_res)); } else { - PyErr_SetString(PyExc_TypeError, "invalid type assigned to expression (corruption?)"); - return(NULL); + PyErr_SetString( + PyExc_TypeError, + "invalid type assigned to expression (corruption?)"); + return (NULL); } } array_dims = malloc(sizeof(*array_dims) * ndim); @@ -507,9 +587,10 @@ PyObject *wrapper_dspaces_ops_calc(PyObject *clientppy, PyObject *exprppy) } else if(etype == DS_VAL_REAL) { typenum = NPY_FLOAT64; } else { - PyErr_SetString(PyExc_TypeError, "invalid type assigned to expression (corruption?)"); - return(NULL); + PyErr_SetString(PyExc_TypeError, + "invalid type assigned to expression (corruption?)"); + return (NULL); } arr = PyArray_SimpleNewFromData(ndim, array_dims, typenum, result_buf); - return(arr); + return (arr); } diff --git a/bindings/python/dspaces_wrapper.h b/bindings/python/dspaces_wrapper.h index f59e53c3..07759202 100644 --- a/bindings/python/dspaces_wrapper.h +++ b/bindings/python/dspaces_wrapper.h @@ -1,12 +1,17 @@ +#include + PyObject *wrapper_dspaces_init(int rank); PyObject *wrapper_dspaces_init_mpi(PyObject *commpy); -PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, int rank); +PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, + int rank); -PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, PyObject *commpy); +PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, + PyObject *commpy); -PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, const char *conf); +PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, + const char *conf); void wrapper_dspaces_fini(PyObject *clientppy); @@ -22,16 +27,23 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, PyObject *dtype, int timeout); PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, - int version, PyObject *lbt, PyObject *ubt, - PyObject *fn, const char *fn_name); + int version, PyObject *lbt, PyObject *ubt, + PyObject *fn, const char *fn_name); + +void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, + PyObject *gdimt); + +PyObject *wrapper_dspaces_get_vars(PyObject *clientppy); -void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, PyObject *gdimt); +PyObject *wrapper_dspaces_get_var_objs(PyObject *clientppy, const char *name); PyObject *wrapper_dspaces_ops_new_iconst(long val); PyObject *wrapper_dspaces_ops_new_rconst(double val); -PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, int version, PyObject *lbt, PyObject *ubt, PyObject *dtype); +PyObject *wrapper_dspaces_ops_new_obj(PyObject *clientppy, const char *name, + int version, PyObject *lbt, PyObject *ubt, + PyObject *dtype); PyObject *wrapper_dspaces_op_new_add(PyObject *exprppy1, PyObject *exprppy2); From 6705326f68287f380adfca475974018b98ba461b Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Tue, 7 May 2024 11:47:53 -0400 Subject: [PATCH 02/10] Reverse index order in python bindings --- bindings/python/dspaces_wrapper.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bindings/python/dspaces_wrapper.c b/bindings/python/dspaces_wrapper.c index a77d2632..2126426b 100644 --- a/bindings/python/dspaces_wrapper.c +++ b/bindings/python/dspaces_wrapper.c @@ -184,10 +184,11 @@ void wrapper_dspaces_put(PyObject *clientppy, PyObject *obj, const char *name, PyObject *item; int i; + //Reverse order of indices for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(offset, i); - lb[i] = PyLong_AsLong(item); - ub[i] = lb[i] + ((long)shape[i] - 1); + lb[(ndim-1) - i] = PyLong_AsLong(item); + ub[(ndim-1) - i] = lb[(ndim-1) - i] + ((long)shape[i] - 1); } dspaces_put_tag(*clientp, name, version, size, tag, ndim, lb, ub, data); @@ -210,11 +211,12 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, npy_intp dims[ndim]; int i; + //Reverse order of indices for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(lbt, i); - lb[i] = PyLong_AsLong(item); + lb[(ndim-1) - i] = PyLong_AsLong(item); item = PyTuple_GetItem(ubt, i); - ub[i] = PyLong_AsLong(item); + ub[(ndim-1) - i] = PyLong_AsLong(item); } Py_BEGIN_ALLOW_THREADS dspaces_aget(*clientp, name, version, ndim, lb, ub, @@ -234,7 +236,7 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, } for(i = 0; i < ndim; i++) { - dims[i] = ((ub[i] - lb[i]) + 1); + dims[(ndim-1) - i] = ((ub[i] - lb[i]) + 1); } arr = PyArray_NewFromDescr(&PyArray_Type, descr, ndim, dims, NULL, data, 0, @@ -271,11 +273,12 @@ PyObject *wrapper_dspaces_pexec(PyObject *clientppy, const char *name, lb = malloc(sizeof(*lb) * ndim); ub = malloc(sizeof(*ub) * ndim); + //Reverse order of indices for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(lbt, i); - lb[i] = PyLong_AsLong(item); + lb[(ndim-1) - i] = PyLong_AsLong(item); item = PyTuple_GetItem(ubt, i); - ub[i] = PyLong_AsLong(item); + ub[(ndim-1) - i] = PyLong_AsLong(item); } if(!PyBytes_Check(fn)) { @@ -311,7 +314,7 @@ void wrapper_dspaces_define_gdim(PyObject *clientppy, const char *name, for(i = 0; i < ndim; i++) { item = PyTuple_GetItem(gdimt, i); - gdim[i] = PyLong_AsLong(item); + gdim[(ndim-1) - i] = PyLong_AsLong(item); } dspaces_define_gdim(*clientp, name, ndim, gdim); From 093bb653ba2e5c1f5318bcd337f861153fbe4063 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Wed, 8 May 2024 09:22:39 -0400 Subject: [PATCH 03/10] Release GIL for more dspaces operations in python bindings --- bindings/python/dspaces_wrapper.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/bindings/python/dspaces_wrapper.c b/bindings/python/dspaces_wrapper.c index 2126426b..619bd0f7 100644 --- a/bindings/python/dspaces_wrapper.c +++ b/bindings/python/dspaces_wrapper.c @@ -22,7 +22,11 @@ PyObject *wrapper_dspaces_init(int rank) clientp = malloc(sizeof(*clientp)); + // clang-format off + Py_BEGIN_ALLOW_THREADS ret = dspaces_init(rank, clientp); + Py_END_ALLOW_THREADS + // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); @@ -50,7 +54,11 @@ PyObject *wrapper_dspaces_init_mpi(PyObject *commpy) } clientp = malloc(sizeof(*clientp)); + // clang-format off + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_mpi(*comm_p, clientp); + Py_END_ALLOW_THREADS + // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); @@ -74,7 +82,11 @@ PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, clientp = malloc(sizeof(*clientp)); + // clang-format off + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan(listen_str, conn, rank, clientp); + Py_END_ALLOW_THREADS + // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_wan() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); @@ -103,7 +115,11 @@ PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, } clientp = malloc(sizeof(*clientp)); + // clang-format off + Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan_mpi(listen_str, conn, *comm_p, clientp); + Py_END_ALLOW_THREADS + // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_wan_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); @@ -132,7 +148,11 @@ PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, } serverp = malloc(sizeof(*serverp)); + // clang-format off + Py_BEGIN_ALLOW_THREADS ret = dspaces_server_init(listen_str, *comm_p, conf, serverp); + Py_END_ALLOW_THREADS + // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); PyErr_SetString(PyExc_RuntimeError, err_str); @@ -190,7 +210,12 @@ void wrapper_dspaces_put(PyObject *clientppy, PyObject *obj, const char *name, lb[(ndim-1) - i] = PyLong_AsLong(item); ub[(ndim-1) - i] = lb[(ndim-1) - i] + ((long)shape[i] - 1); } + + // clang-format off + Py_BEGIN_ALLOW_THREADS dspaces_put_tag(*clientp, name, version, size, tag, ndim, lb, ub, data); + Py_END_ALLOW_THREADS + // clang-format on return; } @@ -219,9 +244,13 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, ub[(ndim-1) - i] = PyLong_AsLong(item); } - Py_BEGIN_ALLOW_THREADS dspaces_aget(*clientp, name, version, ndim, lb, ub, + + // clang-format off + Py_BEGIN_ALLOW_THREADS + dspaces_aget(*clientp, name, version, ndim, lb, ub, &data, &tag, timeout); Py_END_ALLOW_THREADS + // clang-format on if(data == NULL) { From cc8602e524888cea58bc28995231c32f4234b20c Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Tue, 14 May 2024 13:50:43 -0400 Subject: [PATCH 04/10] Zero-out bulk_in_t during get_data initialization to avoid setting flags. --- src/dspaces-client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dspaces-client.c b/src/dspaces-client.c index 70bae01b..55661ade 100644 --- a/src/dspaces-client.c +++ b/src/dspaces-client.c @@ -1293,7 +1293,7 @@ static int get_data(dspaces_client_t client, int num_odscs, int ret; hg_return_t hret; - in = (bulk_in_t *)malloc(sizeof(bulk_in_t) * num_odscs); + in = (bulk_in_t *)calloc(sizeof(bulk_in_t), num_odscs); od = malloc(num_odscs * sizeof(struct obj_data *)); hndl = (hg_handle_t *)malloc(sizeof(hg_handle_t) * num_odscs); serv_req = (margo_request *)malloc(sizeof(margo_request) * num_odscs); From 9156d434178631b508558c85ef338005a8e7ddb0 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Tue, 14 May 2024 14:29:45 -0400 Subject: [PATCH 05/10] Added some query debugging --- src/dspaces-client.c | 2 ++ src/dspaces-server.c | 1 + 2 files changed, 3 insertions(+) diff --git a/src/dspaces-client.c b/src/dspaces-client.c index 55661ade..300478f5 100644 --- a/src/dspaces-client.c +++ b/src/dspaces-client.c @@ -1558,6 +1558,7 @@ int dspaces_put_local(dspaces_client_t client, const char *var_name, margo_destroy(handle); return dspaces_ERR_MERCURY; } + DEBUG_OUT("RPC sent, awaiting response.\n"); hret = margo_get_output(handle, &out); if(hret != HG_SUCCESS) { @@ -1610,6 +1611,7 @@ static int get_odscs(dspaces_client_t client, obj_descriptor *odsc, int timeout, margo_destroy(handle); return (0); } + DEBUG_OUT("RPC sent, awaiting reply.\n"); hret = margo_get_output(handle, &out); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: %s: margo_get_output() failed with %d.\n", diff --git a/src/dspaces-server.c b/src/dspaces-server.c index 228b2911..6d0127a2 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -2361,6 +2361,7 @@ static void query_rpc(hg_handle_t handle) get_query_odscs(server, &in, timeout, &results, req_id); out.odsc_list.raw_odsc = (char *)results; + DEBUG_OUT("Responding with %li results.\n", out.odsc_list.size / sizeof(obj_descriptor)); margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); From 816e2bd83bc8fd86efcc47a8eae8cb51611db98c Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Tue, 14 May 2024 14:47:58 -0400 Subject: [PATCH 06/10] More debug --- src/dspaces-client.c | 7 +++++-- src/dspaces-server.c | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/dspaces-client.c b/src/dspaces-client.c index 300478f5..171e04b2 100644 --- a/src/dspaces-client.c +++ b/src/dspaces-client.c @@ -1590,20 +1590,23 @@ static int get_odscs(dspaces_client_t client, obj_descriptor *odsc, int timeout, in.odsc_gdim.size = sizeof(*odsc); in.odsc_gdim.raw_odsc = (char *)odsc; in.param = timeout; - + + DEBUG_OUT("starting query.\n"); set_global_dimension(&(client->dcg->gdim_list), odsc->name, &(client->dcg->default_gdim), &od_gdim); in.odsc_gdim.gdim_size = sizeof(od_gdim); in.odsc_gdim.raw_gdim = (char *)(&od_gdim); + DEBUG_OUT("Found gdims.\n"); get_server_address(client, &server_addr); - + hret = margo_create(client->mid, server_addr, client->query_id, &handle); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: %s: margo_create() failed with %d.\n", __func__, hret); return (0); } + DEBUG_OUT("Forwarding RPC.\n"); hret = margo_forward(handle, &in); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: %s: margo_forward() failed with %d.\n", diff --git a/src/dspaces-server.c b/src/dspaces-server.c index 6d0127a2..b469f3bf 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -2361,7 +2361,7 @@ static void query_rpc(hg_handle_t handle) get_query_odscs(server, &in, timeout, &results, req_id); out.odsc_list.raw_odsc = (char *)results; - DEBUG_OUT("Responding with %li results.\n", out.odsc_list.size / sizeof(obj_descriptor)); + DEBUG_OUT("Responding with %li result(s).\n", out.odsc_list.size / sizeof(obj_descriptor)); margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); From bcbd42ad324622873c5855f4d869ae28836848a8 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Wed, 15 May 2024 16:56:26 -0400 Subject: [PATCH 07/10] (temp) do not release GIL in python bindings --- bindings/python/dspaces_wrapper.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bindings/python/dspaces_wrapper.c b/bindings/python/dspaces_wrapper.c index 619bd0f7..d65aacca 100644 --- a/bindings/python/dspaces_wrapper.c +++ b/bindings/python/dspaces_wrapper.c @@ -23,9 +23,9 @@ PyObject *wrapper_dspaces_init(int rank) clientp = malloc(sizeof(*clientp)); // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS ret = dspaces_init(rank, clientp); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init() failed with %i", ret); @@ -55,9 +55,9 @@ PyObject *wrapper_dspaces_init_mpi(PyObject *commpy) clientp = malloc(sizeof(*clientp)); // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS ret = dspaces_init_mpi(*comm_p, clientp); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); @@ -83,9 +83,9 @@ PyObject *wrapper_dspaces_init_wan(const char *listen_str, const char *conn, clientp = malloc(sizeof(*clientp)); // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan(listen_str, conn, rank, clientp); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_wan() failed with %i", ret); @@ -116,9 +116,9 @@ PyObject *wrapper_dspaces_init_wan_mpi(const char *listen_str, const char *conn, clientp = malloc(sizeof(*clientp)); // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS ret = dspaces_init_wan_mpi(listen_str, conn, *comm_p, clientp); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_wan_mpi() failed with %i", ret); @@ -149,9 +149,9 @@ PyObject *wrapper_dspaces_server_init(const char *listen_str, PyObject *commpy, serverp = malloc(sizeof(*serverp)); // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS ret = dspaces_server_init(listen_str, *comm_p, conf, serverp); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(ret != 0) { sprintf(err_str, "dspaces_init_mpi() failed with %i", ret); @@ -212,9 +212,9 @@ void wrapper_dspaces_put(PyObject *clientppy, PyObject *obj, const char *name, } // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS dspaces_put_tag(*clientp, name, version, size, tag, ndim, lb, ub, data); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on return; @@ -246,10 +246,10 @@ PyObject *wrapper_dspaces_get(PyObject *clientppy, const char *name, // clang-format off - Py_BEGIN_ALLOW_THREADS + //Py_BEGIN_ALLOW_THREADS dspaces_aget(*clientp, name, version, ndim, lb, ub, &data, &tag, timeout); - Py_END_ALLOW_THREADS + //Py_END_ALLOW_THREADS // clang-format on if(data == NULL) From 4ab00ba6a68e9c55e6f1df846e073c4d7ee045ce Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Fri, 17 May 2024 00:13:22 -0400 Subject: [PATCH 08/10] Handle missing data inputs for pexec --- src/dspaces-server.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/dspaces-server.c b/src/dspaces-server.c index b469f3bf..85f09497 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -3182,6 +3182,15 @@ static void pexec_rpc(hg_handle_t handle) route_request(server, &in_odsc, &(server->dsg->default_gdim)); from_obj = ls_find(server->dsg->ls, &in_odsc); + if(!from_obj) { + DEBUG_OUT("could not find input object\n"); + out.length = 0; + out.handle = 0; + margo_respond(handle, &out); + margo_free_input(handle, &in); + margo_destroy(handle); + return; + } array = build_ndarray_from_od(from_obj); // Race condition? Protect with mutex? @@ -3224,6 +3233,7 @@ static void pexec_rpc(hg_handle_t handle) fprintf(stderr, "ERROR: (%s): margo_bulk_create failed with %d.\n", __func__, hret); out.length = 0; + out.handle = 0; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); From 6f5602108e5037196c1808b86f609acb92507151 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Fri, 17 May 2024 12:52:02 -0400 Subject: [PATCH 09/10] Coalesce local storage during pexec calls --- include/ss_data.h | 1 + src/dspaces-server.c | 32 +++++++++++++++++++++++++++----- src/ss_data.c | 29 +++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/include/ss_data.h b/include/ss_data.h index c0c388b1..eab6e16b 100644 --- a/include/ss_data.h +++ b/include/ss_data.h @@ -440,6 +440,7 @@ struct obj_data *ls_find_no_version(ss_storage *, obj_descriptor *); int ls_get_var_names(ss_storage *, char ***); int ls_find_all_no_version(ss_storage *ls, const char *var_name, obj_descriptor ***odscs); +int ls_find_all(ss_storage *ls, obj_descriptor *odsc, struct obj_data ***ods); struct obj_data *obj_data_alloc(obj_descriptor *); struct obj_data *obj_data_alloc_no_data(obj_descriptor *, void *); diff --git a/src/dspaces-server.c b/src/dspaces-server.c index 85f09497..bdddf6d6 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -3127,14 +3127,16 @@ static void pexec_rpc(hg_handle_t handle) pexec_out_t out; hg_return_t hret; hg_bulk_t bulk_handle; - obj_descriptor in_odsc; + obj_descriptor in_odsc, odsc; hg_size_t rdma_size; void *fn = NULL, *res_data; - struct obj_data *od, *from_obj; + struct obj_data *od, *arg_obj, **od_tab, **from_objs = NULL; + int num_obj; PyObject *array, *fnp, *arg, *pres, *pres_bytes; static PyObject *pklmod = NULL; ABT_cond cond; ABT_mutex mtx; + int i; hret = margo_get_input(handle, &in); if(hret != HG_SUCCESS) { @@ -3181,8 +3183,25 @@ static void pexec_rpc(hg_handle_t handle) margo_bulk_free(bulk_handle); route_request(server, &in_odsc, &(server->dsg->default_gdim)); - from_obj = ls_find(server->dsg->ls, &in_odsc); - if(!from_obj) { + ABT_mutex_lock(server->ls_mutex); + num_obj = ls_find_all(server->dsg->ls, &in_odsc, &from_objs); + if(num_obj > 0) { + DEBUG_OUT("found %i objects in local storage to populate input\n", num_obj); + arg_obj = obj_data_alloc(&in_odsc); + od_tab = malloc(num_obj * sizeof(*od_tab)); + for(i = 0; i < num_obj; i++) { + // Can we skip the intermediate copy? + odsc = from_objs[i]->obj_desc; + bbox_intersect(&in_odsc.bb, &odsc.bb, &odsc.bb); + od_tab[i] = obj_data_alloc(&odsc); + ssd_copy(od_tab[i], from_objs[i]); + ssd_copy(arg_obj, od_tab[i]); + obj_data_free(od_tab[i]); + } + free(od_tab); + } + ABT_mutex_unlock(server->ls_mutex); + if(num_obj < 1) { DEBUG_OUT("could not find input object\n"); out.length = 0; out.handle = 0; @@ -3190,8 +3209,10 @@ static void pexec_rpc(hg_handle_t handle) margo_free_input(handle, &in); margo_destroy(handle); return; + } else if(from_objs) { + free(from_objs); } - array = build_ndarray_from_od(from_obj); + array = build_ndarray_from_od(arg_obj); // Race condition? Protect with mutex? if((pklmod == NULL) && @@ -3273,6 +3294,7 @@ static void pexec_rpc(hg_handle_t handle) DEBUG_OUT("done with pexec handling\n"); Py_XDECREF(array); + obj_data_free(arg_obj); } DEFINE_MARGO_RPC_HANDLER(pexec_rpc) #endif // DSPACES_HAVE_PYTHON diff --git a/src/ss_data.c b/src/ss_data.c index 913e8d70..a6fce07c 100644 --- a/src/ss_data.c +++ b/src/ss_data.c @@ -1052,6 +1052,35 @@ int ls_find_all_no_version(ss_storage *ls, const char *var_name, return (n); } +int ls_find_all(ss_storage *ls, obj_descriptor *odsc, struct obj_data ***ods) +{ + struct obj_data *od; + struct list_head *list; + int i, index, n = 0; + long num_elem; + struct bbox isect; + + num_elem = bbox_volume(&odsc->bb); + *ods = malloc(sizeof(**ods) * ls->num_obj); + index = odsc->version % ls->size_hash; + list = &ls->obj_hash[index]; + + list_for_each_entry(od, list, struct obj_data, obj_entry) + { + if(obj_desc_equals_intersect(odsc, &od->obj_desc)) { + (*ods)[n++] = od; + bbox_intersect(&odsc->bb, &od->obj_desc.bb, &isect); + num_elem -= bbox_volume(&isect); + } + } + if(num_elem != 0) { + free(*ods); + return(-1); + } + + return(n); +} + /* * Do two object descriptors have the same name and version? */ From 337012372aabf7183a12338affa31d2b0707fa0d Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Fri, 17 May 2024 16:59:59 -0400 Subject: [PATCH 10/10] Extend static modules --- src/dspaces-server.c | 54 ++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/src/dspaces-server.c b/src/dspaces-server.c index bdddf6d6..2fe2d781 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -1016,12 +1016,18 @@ static void *bootstrap_python() static int dspaces_init_py_mods(dspaces_provider_t server, struct dspaces_module **pmodsp) { + static const char *static_pmods[][2] = { + {"goes17", "s3nc_mod"}, + {"planetary", "azure_mod"}, + {"cmips3", "s3cmip_mod"} + }; char *pypath = getenv("PYTHONPATH"); char *new_pypath; int pypath_len; struct dspaces_module *pmods; - int npmods = 1; + int npmods = sizeof(static_pmods) / sizeof(static_pmods[0]); PyObject *pName; + int i; pypath_len = strlen(xstr(DSPACES_MOD_DIR)) + 1; if(pypath) { @@ -1039,18 +1045,19 @@ static int dspaces_init_py_mods(dspaces_provider_t server, bootstrap_python(); pmods = malloc(sizeof(*pmods) * npmods); - pmods[0].name = strdup("goes17"); - pmods[0].type = DSPACES_MOD_PY; - pName = PyUnicode_DecodeFSDefault("s3nc_mod"); - pmods[0].pModule = PyImport_Import(pName); - if(pmods[0].pModule == NULL) { - fprintf(stderr, - "WARNING: could not load s3nc mod from %s. File missing? Any " - "s3nc accesses will fail.\n", - xstr(DSPACES_MOD_DIR)); - } - Py_DECREF(pName); - + for(i = 0; i < npmods; i++) { + pmods[i].name = strdup(static_pmods[i][0]); + pmods[i].type = DSPACES_MOD_PY; + pName = PyUnicode_DecodeFSDefault(static_pmods[i][1]); + pmods[i].pModule = PyImport_Import(pName); + if(pmods[0].pModule == NULL) { + fprintf(stderr, + "WARNING: could not load %s mod from %s. File missing? Any " + "%s accesses will fail.\n", + static_pmods[i][1], xstr(DSPACES_MOD_DIR), pmods[i].name); + } + Py_DECREF(pName); + } free(new_pypath); *pmodsp = pmods; @@ -2095,22 +2102,31 @@ static void free_arg_list(struct dspaces_module_args *args, int len) static void route_request(dspaces_provider_t server, obj_descriptor *odsc, struct global_dimension *gdim) { + static const char *module_nspaces[][2] = { + {"goes17\\", "goes17"}, + {"cmip6-planetary\\", "planetary"}, + {"cmip6-s3\\", "cmips3"}, + {NULL, NULL} + }; const char *s3nc_nspace = "goes17\\"; + const char *azure_nspace = "cmip6-planetary\\"; + const char *s3cmip_nspace = "cmip6-s3\\"; struct dspaces_module_args *args; struct dspaces_module_ret *res = NULL; struct obj_data *od; obj_descriptor *od_odsc; + char **mod_desc; int nargs; int i; DEBUG_OUT("Routing '%s'\n", odsc->name); - if(strstr(odsc->name, s3nc_nspace) == odsc->name) { - nargs = build_module_args_from_odsc(odsc, &args); - res = dspaces_module_exec(server, "goes17", "query", args, nargs, - DSPACES_MOD_RET_ARRAY); - free_arg_list(args, nargs); - free(args); + for(mod_desc = (char **)module_nspaces[0]; mod_desc[0] != NULL; mod_desc+=2) { + if(strstr(odsc->name, mod_desc[0]) == odsc->name) { + nargs = build_module_args_from_odsc(odsc, &args); + res = dspaces_module_exec(server, mod_desc[1], "query", args, + nargs, DSPACES_MOD_RET_ARRAY); + } } if(res) {