diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..d344d6f9 --- /dev/null +++ b/.clang-format @@ -0,0 +1,6 @@ +Language: Cpp +BasedOnStyle: LLVM +BreakBeforeBraces: Linux +IndentWidth: 4 +ContinuationIndentWidth: 4 +SpaceBeforeParens: Never diff --git a/bindings/fortran/dspaces_f2c.c b/bindings/fortran/dspaces_f2c.c index 9d924de9..9e5839e6 100644 --- a/bindings/fortran/dspaces_f2c.c +++ b/bindings/fortran/dspaces_f2c.c @@ -1,16 +1,18 @@ -#include #include +#include #if defined(__cplusplus) extern "C" { #endif -void FC_GLOBAL(dspaces_init_f2c, DSPACES_INIT_F2C)(int *rank, dspaces_client_t *client, int *ierr) +void FC_GLOBAL(dspaces_init_f2c, + DSPACES_INIT_F2C)(int *rank, dspaces_client_t *client, int *ierr) { *ierr = dspaces_init(*rank, client); } -void FC_GLOBAL(dspaces_fini_f2c, DSPACES_FINI_F2C)(dspaces_client_t *client, int *ierr) +void FC_GLOBAL(dspaces_fini_f2c, DSPACES_FINI_F2C)(dspaces_client_t *client, + int *ierr) { *ierr = dspaces_fini(*client); } @@ -20,19 +22,34 @@ void FC_GLOBAL(dspaces_kill_f2c, DSPACES_KILL_F2C)(dspaces_client_t *client) dspaces_kill(*client); } -void FC_GLOBAL(dspaces_get_f2c, DSPACES_GET_F2C)(dspaces_client_t *client, const char *name, uint32_t *version, uint32_t *size, uint32_t *ndim, uint64_t *lb, uint64_t *ub, uint32_t *data, int *timeout, int *ierr) +void FC_GLOBAL(dspaces_get_f2c, + DSPACES_GET_F2C)(dspaces_client_t *client, const char *name, + uint32_t *version, uint32_t *size, + uint32_t *ndim, uint64_t *lb, uint64_t *ub, + uint32_t *data, int *timeout, int *ierr) { - *ierr = dspaces_get(*client, name, *version, *size, *ndim, lb, ub, data, *timeout); + *ierr = dspaces_get(*client, name, *version, *size, *ndim, lb, ub, data, + *timeout); } -void FC_GLOBAL(dspaces_put_f2c, DSPACES_PUT_F2C)(dspaces_client_t *client, const char *name, uint32_t *version, uint32_t *size, uint32_t *ndim, uint64_t *lb, uint64_t *ub, float *data, int *ierr) +void FC_GLOBAL(dspaces_put_f2c, + DSPACES_PUT_F2C)(dspaces_client_t *client, const char *name, + uint32_t *version, uint32_t *size, + uint32_t *ndim, uint64_t *lb, uint64_t *ub, + float *data, int *ierr) { *ierr = dspaces_put(*client, name, *version, *size, *ndim, lb, ub, data); } -void FC_GLOBAL(dspaces_put_local_f2c, DSPACES_PUT_LOCAL_F2C)(dspaces_client_t *client, const char *name, uint32_t *version, uint32_t *size, uint32_t *ndim, uint64_t *lb, uint64_t *ub, float *data, int *ierr) +void FC_GLOBAL(dspaces_put_local_f2c, + DSPACES_PUT_LOCAL_F2C)(dspaces_client_t *client, + const char *name, uint32_t *version, + uint32_t *size, uint32_t *ndim, + uint64_t *lb, uint64_t *ub, float *data, + int *ierr) { - *ierr = dspaces_put_local(*client, name, *version, *size, *ndim, lb, ub, data); + *ierr = + dspaces_put_local(*client, name, *version, *size, *ndim, lb, ub, data); } #if defined(__cplusplus) diff --git a/src/bbox.c b/src/bbox.c index 3d366245..7ed8f5bd 100644 --- a/src/bbox.c +++ b/src/bbox.c @@ -1,51 +1,55 @@ /* - * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers University - * All rights reserved. + * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers + * University All rights reserved. * - * Redistribution and use in source and binary forms, with or without modification, are permitted provided - * that the following conditions are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * - Redistributions of source code must retain the above copyright notice, this list of conditions and - * the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and - * the following disclaimer in the documentation and/or other materials provided with the distribution. - * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers University, nor the names of its - * contributors may be used to endorse or promote products derived from this software without specific prior - * written permission. + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers + * University, nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written + * permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* -* Ciprian Docan (2009) TASSL Rutgers University -* docan@cac.rutgers.edu -* Tong Jin (2011) TASSL Rutgers University -* tjin@cac.rutgers.edu -*/ + * Ciprian Docan (2009) TASSL Rutgers University + * docan@cac.rutgers.edu + * Tong Jin (2011) TASSL Rutgers University + * tjin@cac.rutgers.edu + */ #include "bbox.h" -#include "sfc.h" #include "queue.h" +#include "sfc.h" -//static inline unsigned int -static inline uint64_t -coord_dist(struct coord *c0, struct coord *c1, int dim) +// static inline unsigned int +static inline uint64_t coord_dist(struct coord *c0, struct coord *c1, int dim) { - return (c1->c[dim] - c0->c[dim] + 1); + return (c1->c[dim] - c0->c[dim] + 1); } -//int bbox_dist(struct bbox *bb, int dim) +// int bbox_dist(struct bbox *bb, int dim) uint64_t bbox_dist(struct bbox *bb, int dim) { - return coord_dist(&bb->lb, &bb->ub, dim); + return coord_dist(&bb->lb, &bb->ub, dim); } /* @@ -61,7 +65,7 @@ void bbox_divide_in2_ondim(const struct bbox *b0, struct bbox *b_tab, int dim) b_tab[1] = b0[0]; b_tab[0].ub.c[dim] = n; - b_tab[1].lb.c[dim] = n+1; + b_tab[1].lb.c[dim] = n + 1; } /* @@ -71,19 +75,18 @@ void bbox_divide_in2_ondim(const struct bbox *b0, struct bbox *b_tab, int dim) void bbox_divide(struct bbox *b0, struct bbox *b_tab) { int ndims = b0->num_dims; - int num_subbox = 1 << ndims; //number of sub n-dimensions bbox + int num_subbox = 1 << ndims; // number of sub n-dimensions bbox int i, j, n = 0; - for(i = 0; i < num_subbox; i++){ + for(i = 0; i < num_subbox; i++) { j = 0; b_tab[i].num_dims = b0->num_dims; - while(j < ndims){ - n = (b0->lb.c[j] + b0->ub.c[j]) / 2; //the middle point of bounding - if((i & (1 << j))==0){ + while(j < ndims) { + n = (b0->lb.c[j] + b0->ub.c[j]) / 2; // the middle point of bounding + if((i & (1 << j)) == 0) { b_tab[i].lb.c[j] = b0->lb.c[j]; b_tab[i].ub.c[j] = n; - } - else{ + } else { b_tab[i].lb.c[j] = n + 1; b_tab[i].ub.c[j] = b0->ub.c[j]; } @@ -92,43 +95,43 @@ void bbox_divide(struct bbox *b0, struct bbox *b_tab) } } -/* - Test if bounding box b0 includes b1 along dimension dim. +/* + Test if bounding box b0 includes b1 along dimension dim. */ -static inline int -bbox_include_ondim(const struct bbox *b0, const struct bbox *b1, int dim) +static inline int bbox_include_ondim(const struct bbox *b0, + const struct bbox *b1, int dim) { - if ((b0->lb.c[dim] <= b1->lb.c[dim]) && - (b0->ub.c[dim] >= b1->ub.c[dim]) ) - return 1; - else return 0; + if((b0->lb.c[dim] <= b1->lb.c[dim]) && (b0->ub.c[dim] >= b1->ub.c[dim])) + return 1; + else + return 0; } -/* +/* Test if bounding box b0 includes b1 (test on all dimensions). */ int bbox_include(const struct bbox *b0, const struct bbox *b1) { int i; - for(i = 0; i < b0->num_dims; i++){ + for(i = 0; i < b0->num_dims; i++) { if(!bbox_include_ondim(b0, b1, i)) return 0; - } + } return 1; } /* Test if bounding boxes b0 and b1 intersect along dimension dim. */ -static int bbox_intersect_ondim(const struct bbox *b0, const struct bbox *b1, int dim) +static int bbox_intersect_ondim(const struct bbox *b0, const struct bbox *b1, + int dim) { - if ((b0->lb.c[dim] <= b1->lb.c[dim] && - b1->lb.c[dim] <= b0->ub.c[dim]) || - (b1->lb.c[dim] <= b0->lb.c[dim] && - b0->lb.c[dim] <= b1->ub.c[dim])) - return 1; - else return 0; + if((b0->lb.c[dim] <= b1->lb.c[dim] && b1->lb.c[dim] <= b0->ub.c[dim]) || + (b1->lb.c[dim] <= b0->lb.c[dim] && b0->lb.c[dim] <= b1->ub.c[dim])) + return 1; + else + return 0; } /* @@ -138,11 +141,11 @@ int bbox_does_intersect(const struct bbox *b0, const struct bbox *b1) { int i; - for(i = 0; i < b0->num_dims; i++){ + for(i = 0; i < b0->num_dims; i++) { if(!bbox_intersect_ondim(b0, b1, i)) return 0; } - + return 1; } @@ -150,15 +153,16 @@ int bbox_does_intersect(const struct bbox *b0, const struct bbox *b1) Compute the intersection of bounding boxes b0 and b1, and store it on b2. Implicit assumption: b0 and b1 intersect. */ -void bbox_intersect(const struct bbox *b0, const struct bbox *b1, struct bbox *b2) +void bbox_intersect(const struct bbox *b0, const struct bbox *b1, + struct bbox *b2) { - int i; + int i; - b2->num_dims = b0->num_dims; - for (i = 0; i < b0->num_dims; i++) { - b2->lb.c[i] = max(b0->lb.c[i], b1->lb.c[i]); - b2->ub.c[i] = min(b0->ub.c[i], b1->ub.c[i]); - } + b2->num_dims = b0->num_dims; + for(i = 0; i < b0->num_dims; i++) { + b2->lb.c[i] = max(b0->lb.c[i], b1->lb.c[i]); + b2->ub.c[i] = min(b0->ub.c[i], b1->ub.c[i]); + } } /* @@ -167,10 +171,9 @@ void bbox_intersect(const struct bbox *b0, const struct bbox *b1, struct bbox *b int bbox_equals(const struct bbox *bb0, const struct bbox *bb1) { int i; - if(bb0->num_dims == bb1->num_dims){ - for(i = 0; i < bb0->num_dims; i++){ - if((bb0->lb.c[i] != bb1->lb.c[i]) || - (bb0->ub.c[i] != bb1->ub.c[i])) + if(bb0->num_dims == bb1->num_dims) { + for(i = 0; i < bb0->num_dims; i++) { + if((bb0->lb.c[i] != bb1->lb.c[i]) || (bb0->ub.c[i] != bb1->ub.c[i])) return 0; } return 1; @@ -184,7 +187,7 @@ uint64_t bbox_volume(struct bbox *bb) int ndims = bb->num_dims; int i; - for(i = 0; i < ndims; i++){ + for(i = 0; i < ndims; i++) { n = n * coord_dist(&bb->lb, &bb->ub, i); } return n; @@ -192,14 +195,14 @@ uint64_t bbox_volume(struct bbox *bb) static int compute_bits(uint64_t n) { - int nr_bits = 0; + int nr_bits = 0; - while (n) { - n = n >> 1; - nr_bits++; - } + while(n) { + n = n >> 1; + nr_bits++; + } - return nr_bits; + return nr_bits; } /* @@ -210,35 +213,35 @@ static int compute_bits(uint64_t n) static void bbox_flat(struct bbox *bb, struct intv *itv, int bpd) { int dims = bb->num_dims; - bitmask_t* sfc_coord; + bitmask_t *sfc_coord; int i, j, k; uint64_t index; - //sfc_coord = malloc(sizeof(bitmask_t)*dims); + // sfc_coord = malloc(sizeof(bitmask_t)*dims); /* - bb->lb.c[0], c[1], c[2]...c[dims] + bb->lb.c[0], c[1], c[2]...c[dims] bb->ub.c[0], c[1], c[2]...c[dims] */ - //itv->lb = ~(0UL); //TODO for 64 bits - itv->lb = ~(0ULL); //TODO for 64 bits + // itv->lb = ~(0UL); //TODO for 64 bits + itv->lb = ~(0ULL); // TODO for 64 bits itv->ub = 0; - //initialize sfc_coord: all the possible 2-based number with dims bits - sfc_coord = malloc(sizeof(bitmask_t) * dims); //TODO free resource - for(i = 0; i < (1<lb.c[k]; } - while(j < dims){ - if(i & (1<ub.c[j]; j++; } index = hilbert_c2i(dims, bpd, sfc_coord); - if (index < itv->lb) + if(index < itv->lb) itv->lb = index; - else if (index > itv->ub) + else if(index > itv->ub) itv->ub = index; } free(sfc_coord); @@ -246,87 +249,86 @@ static void bbox_flat(struct bbox *bb, struct intv *itv, int bpd) static int intv_compar(const void *a, const void *b) { - const struct intv *i0 = a, *i1 = b; - - // return (int) (i0->lb - i1->lb); - if (i0->lb < i1->lb) - return -1; - else if (i0->lb > i1->lb) - return 1; - else return 0; + const struct intv *i0 = a, *i1 = b; + + // return (int) (i0->lb - i1->lb); + if(i0->lb < i1->lb) + return -1; + else if(i0->lb > i1->lb) + return 1; + else + return 0; } static uint64_t intv_compact(struct intv *i_tab, uint64_t num_itv) { - uint64_t i, j; - - for (i = 0, j = 1; j < num_itv; j++) { - if ((i_tab[i].ub + 1) == i_tab[j].lb) - i_tab[i].ub = i_tab[j].ub; - else { - i = i + 1; - i_tab[i] = i_tab[j]; - } + uint64_t i, j; + + for(i = 0, j = 1; j < num_itv; j++) { + if((i_tab[i].ub + 1) == i_tab[j].lb) + i_tab[i].ub = i_tab[j].ub; + else { + i = i + 1; + i_tab[i] = i_tab[j]; } + } - return (i+1); + return (i + 1); } /* Find the equivalence in 1d index space using a SFC for a bounding box bb. */ -void bbox_to_intv(const struct bbox *bb, uint64_t dim_virt, int bpd, +void bbox_to_intv(const struct bbox *bb, uint64_t dim_virt, int bpd, struct intv **intv, int *num_intv) { - //const int 1<<(bb->num_dims); + // const int 1<<(bb->num_dims); struct bbox *bb_virt; - struct bbox *b_tab; //the number of b_tab is 2^n + struct bbox *b_tab; // the number of b_tab is 2^n struct queue q_can, q_good; struct intv *i_tab, *i_tmp; - //int max; + // int max; uint64_t max; int i, n; - max = dim_virt; //n is the next power of 2 that includes the user's bbox - bpd = compute_bits(max); //TODO + max = dim_virt; // n is the next power of 2 that includes the user's bbox + bpd = compute_bits(max); // TODO bb_virt = malloc(sizeof(struct bbox)); memset(bb_virt, 0, sizeof(struct bbox)); bb_virt->num_dims = bb->num_dims; - for(i = 0; i < bb->num_dims; i++){ + for(i = 0; i < bb->num_dims; i++) { bb_virt->ub.c[i] = max - 1; } queue_init(&q_can); queue_init(&q_good); - n = 1 << (bb->num_dims); //number of b_tab + n = 1 << (bb->num_dims); // number of b_tab b_tab = malloc(sizeof(struct bbox) * n); - memset(b_tab, 0, sizeof(struct bbox) * n); //TODO sizeof??? + memset(b_tab, 0, sizeof(struct bbox) * n); // TODO sizeof??? queue_enqueue(&q_can, bb_virt); - while(! queue_is_empty(&q_can)){ + while(!queue_is_empty(&q_can)) { bb_virt = (struct bbox *)queue_dequeue(&q_can); - if(bbox_include(bb, bb_virt)){ + if(bbox_include(bb, bb_virt)) { i_tmp = malloc(sizeof(struct intv)); bbox_flat(bb_virt, i_tmp, bpd); queue_enqueue(&q_good, i_tmp); free(bb_virt); - } - else if(bbox_does_intersect(bb, bb_virt)){ + } else if(bbox_does_intersect(bb, bb_virt)) { bbox_divide(bb_virt, b_tab); free(bb_virt); - for(i = 0; i < n; i++){ + for(i = 0; i < n; i++) { bb_virt = malloc(sizeof(struct bbox)); *bb_virt = b_tab[i]; queue_enqueue(&q_can, bb_virt); } - } - else + } else free(bb_virt); } free(b_tab); @@ -334,7 +336,7 @@ void bbox_to_intv(const struct bbox *bb, uint64_t dim_virt, int bpd, i_tab = malloc(n * sizeof(struct intv)); n = 0; - while(!queue_is_empty(&q_good)){ + while(!queue_is_empty(&q_good)) { i_tmp = queue_dequeue(&q_good); i_tab[n++] = *i_tmp; free(i_tmp); @@ -342,17 +344,16 @@ void bbox_to_intv(const struct bbox *bb, uint64_t dim_virt, int bpd, qsort(i_tab, n, sizeof(struct intv), &intv_compar); n = intv_compact(i_tab, n); - i_tab = realloc(i_tab, n*sizeof(struct intv)); + i_tab = realloc(i_tab, n * sizeof(struct intv)); *intv = i_tab; *num_intv = n; } - /* New test ... */ -void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, - struct intv **intv, int *num_intv) +void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, + struct intv **intv, int *num_intv) { struct bbox *bb_tab, *pbb; int bb_size, bb_head, bb_tail; @@ -363,7 +364,7 @@ void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, n = dim_virt; bpd = compute_bits(n); - // bb_size = 4000; //TODO + // bb_size = 4000; //TODO bb_size = 4096; bb_tab = malloc(sizeof(*bb_tab) * bb_size); pbb = &bb_tab[bb_size - 1]; @@ -371,61 +372,62 @@ void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, bb_tail = 0; pbb->num_dims = bb->num_dims; - for(i = 0; i < pbb->num_dims; i++){ + for(i = 0; i < pbb->num_dims; i++) { pbb->lb.c[i] = 0; pbb->ub.c[i] = n - 1; } - //i_size = 4000; + // i_size = 4000; i_size = 4096; i_num = 0; i_tab = malloc(sizeof(*i_tab) * i_size); n = 1 << (bb->num_dims); - while(bb_head != bb_tail){ + while(bb_head != bb_tail) { pbb = &bb_tab[bb_head]; - if(bbox_include(bb, pbb)){ - if(i_num == i_size){ - i_size = i_size + i_size/2; - i_tab = realloc(i_tab, sizeof(*i_tab)*i_size); + if(bbox_include(bb, pbb)) { + if(i_num == i_size) { + i_size = i_size + i_size / 2; + i_tab = realloc(i_tab, sizeof(*i_tab) * i_size); i_resize++; } bbox_flat(pbb, &i_tab[i_num], bpd); i_num++; - } - else if(bbox_does_intersect(bb, pbb)){ - if((bb_tail+n)%bb_size == (bb_head - (bb_head%n))){ - // int bb_nsize = (bb_size + bb_size/2) & (~0x07); /**1 byte is 8 bits**/ - int tmp_size = bb_size + bb_size/2; + } else if(bbox_does_intersect(bb, pbb)) { + if((bb_tail + n) % bb_size == (bb_head - (bb_head % n))) { + // int bb_nsize = (bb_size + bb_size/2) & (~0x07); /**1 byte is + // 8 bits**/ + int tmp_size = bb_size + bb_size / 2; int bb_nsize = tmp_size - tmp_size % (1 << bb->num_dims); - struct bbox *bb_ntab; - int bb_nhead = bb_head - (bb_head % n); - - bb_ntab = malloc(sizeof(*bb_ntab) * bb_nsize); - if (bb_tail > bb_head) { - memcpy(bb_ntab, &bb_tab[bb_nhead], sizeof(*bb_ntab)*(bb_tail-bb_nhead)); - } - else { - memcpy(bb_ntab, &bb_tab[bb_nhead], sizeof(*bb_ntab)*(bb_size-bb_nhead)); - memcpy(&bb_ntab[bb_size-bb_nhead], bb_tab, sizeof(*bb_ntab)*bb_tail); - } - bb_head = bb_head % n; - bb_tail = bb_size - n; - bb_size = bb_nsize; - - free(bb_tab); - bb_tab = bb_ntab; - pbb = &bb_tab[bb_head]; - } + struct bbox *bb_ntab; + int bb_nhead = bb_head - (bb_head % n); + + bb_ntab = malloc(sizeof(*bb_ntab) * bb_nsize); + if(bb_tail > bb_head) { + memcpy(bb_ntab, &bb_tab[bb_nhead], + sizeof(*bb_ntab) * (bb_tail - bb_nhead)); + } else { + memcpy(bb_ntab, &bb_tab[bb_nhead], + sizeof(*bb_ntab) * (bb_size - bb_nhead)); + memcpy(&bb_ntab[bb_size - bb_nhead], bb_tab, + sizeof(*bb_ntab) * bb_tail); + } + bb_head = bb_head % n; + bb_tail = bb_size - n; + bb_size = bb_nsize; + + free(bb_tab); + bb_tab = bb_ntab; + pbb = &bb_tab[bb_head]; + } bbox_divide(pbb, &bb_tab[bb_tail]); - bb_tail = (bb_tail + n) % bb_size; + bb_tail = (bb_tail + n) % bb_size; } - bb_head = (bb_head + 1) % bb_size; + bb_head = (bb_head + 1) % bb_size; } free(bb_tab); - qsort(i_tab, i_num, sizeof(*i_tab), &intv_compar); n = intv_compact(i_tab, i_num); @@ -435,7 +437,6 @@ void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, *num_intv = n; } - /* Translates a bounding box coordinates from global space described by bb_glb to local space. The bounding box for a local space should @@ -443,17 +444,17 @@ void bbox_to_intv2(const struct bbox *bb, uint64_t dim_virt, int bpd, */ void bbox_to_origin(struct bbox *bb, const struct bbox *bb_glb) { - int i; + int i; - if (bb->num_dims != bb_glb->num_dims) { - fprintf(stderr, "ERROR: '%s()': dimensionality mismatch.\n", __func__); - } + if(bb->num_dims != bb_glb->num_dims) { + fprintf(stderr, "ERROR: '%s()': dimensionality mismatch.\n", __func__); + } - bb->num_dims = bb_glb->num_dims; - for (i = 0; i < bb->num_dims; i++) { - bb->lb.c[i] -= bb_glb->lb.c[i]; - bb->ub.c[i] -= bb_glb->lb.c[i]; - } + bb->num_dims = bb_glb->num_dims; + for(i = 0; i < bb->num_dims; i++) { + bb->lb.c[i] -= bb_glb->lb.c[i]; + bb->ub.c[i] -= bb_glb->lb.c[i]; + } } /* @@ -461,29 +462,28 @@ void bbox_to_origin(struct bbox *bb, const struct bbox *bb_glb) */ int intv_do_intersect(struct intv *i0, struct intv *i1) { - if ((i0->lb <= i1->lb && i1->lb <= i0->ub) || - (i1->lb <= i0->lb && i0->lb <= i1->ub)) - return 1; - else return 0; + if((i0->lb <= i1->lb && i1->lb <= i0->ub) || + (i1->lb <= i0->lb && i0->lb <= i1->ub)) + return 1; + else + return 0; } -uint64_t intv_size(struct intv *intv) -{ - return intv->ub - intv->lb + 1; -} +uint64_t intv_size(struct intv *intv) { return intv->ub - intv->lb + 1; } void coord_print(struct coord *c, int num_dims) { - switch (num_dims) { - case 3: - printf("{%" PRIu64 ", %" PRIu64 ", %" PRIu64 "}", c->c[0], c->c[1], c->c[2]); - break; - case 2: - printf("{%" PRIu64 ", %" PRIu64 "}", c->c[0], c->c[1]); - break; - case 1: - printf("{%" PRIu64 "}", c->c[0]); - } + switch(num_dims) { + case 3: + printf("{%" PRIu64 ", %" PRIu64 ", %" PRIu64 "}", c->c[0], c->c[1], + c->c[2]); + break; + case 2: + printf("{%" PRIu64 ", %" PRIu64 "}", c->c[0], c->c[1]); + break; + case 1: + printf("{%" PRIu64 "}", c->c[0]); + } } /* @@ -500,32 +500,32 @@ char *coord_sprint(const struct coord *c, int num_dims) size += snprintf(NULL, 0, "%" PRIu64, c->c[i]); if(i > 0) { } - size += i ? 2 : 0; // account for ", " + size += i ? 2 : 0; // account for ", " } str = malloc(sizeof(*str) * (size + 1)); // add null terminator strcpy(str, "{"); for(i = 0; i < num_dims; i++) { - char *tmp = alloc_sprintf(i?", %" PRIu64 :"%" PRIu64, c->c[i]); + char *tmp = alloc_sprintf(i ? ", %" PRIu64 : "%" PRIu64, c->c[i]); str = str_append(str, tmp); } str = str_append_const(str, "}"); - - return str; + + return str; } void bbox_print(struct bbox *bb) { - printf("{lb = "); - coord_print(&bb->lb, bb->num_dims); - printf(", ub = "); - coord_print(&bb->ub, bb->num_dims); - printf("}"); + printf("{lb = "); + coord_print(&bb->lb, bb->num_dims); + printf(", ub = "); + coord_print(&bb->ub, bb->num_dims); + printf("}"); } -char * bbox_sprint(const struct bbox *bb) +char *bbox_sprint(const struct bbox *bb) { char *str = strdup("{lb = "); - + str = str_append(str, coord_sprint(&bb->lb, bb->num_dims)); str = str_append_const(str, ", ub = "); str = str_append(str, coord_sprint(&bb->ub, bb->num_dims)); diff --git a/src/dspaces-client.c b/src/dspaces-client.c index ee171ac9..a1f9fd52 100644 --- a/src/dspaces-client.c +++ b/src/dspaces-client.c @@ -1,29 +1,31 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "dspaces.h" +#include "gspace.h" +#include "ss_data.h" +#include #include +#include +#include #include #include #include -#include -#include #include -#include -#include -#include -#include "ss_data.h" -#include "dspaces.h" -#include "gspace.h" +#include +#include -#define DEBUG_OUT(args...) \ - do { \ - if(client->f_debug) { \ - fprintf(stderr, "Rank %i: %s, line %i (%s): ", client->rank, __FILE__, __LINE__, __func__); \ - fprintf(stderr, args); \ - } \ - }while(0); +#define DEBUG_OUT(args...) \ + do { \ + if(client->f_debug) { \ + fprintf(stderr, "Rank %i: %s, line %i (%s): ", client->rank, \ + __FILE__, __LINE__, __func__); \ + fprintf(stderr, args); \ + } \ + } while(0); #define SUB_HASH_SIZE 16 @@ -54,9 +56,9 @@ struct dspaces_client { hg_id_t ss_id; hg_id_t drain_id; hg_id_t kill_id; - hg_id_t sub_id; + hg_id_t sub_id; hg_id_t notify_id; - struct dc_gspace *dcg; + struct dc_gspace *dcg; char **server_address; int size_sp; int rank; @@ -88,17 +90,19 @@ static void kill_rpc(hg_handle_t h); DECLARE_MARGO_RPC_HANDLER(notify_rpc); static void notify_rpc(hg_handle_t h); -//round robin fashion -//based on how many clients processes are connected to the server -static hg_return_t get_server_address(dspaces_client_t client, hg_addr_t *server_addr) -{ +// round robin fashion +// based on how many clients processes are connected to the server +static hg_return_t get_server_address(dspaces_client_t client, + hg_addr_t *server_addr) +{ int peer_id = client->rank % client->size_sp; - - return(margo_addr_lookup(client->mid, client->server_address[peer_id], server_addr)); -} + return (margo_addr_lookup(client->mid, client->server_address[peer_id], + server_addr)); +} -static int get_ss_info(dspaces_client_t client) { +static int get_ss_info(dspaces_client_t client) +{ hg_return_t hret; hg_handle_t handle; ss_information out; @@ -113,20 +117,20 @@ static int get_ss_info(dspaces_client_t client) { /* create handle */ hret = margo_create(client->mid, server_addr, client->ss_id, &handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); return dspaces_ERR_MERCURY; } hret = margo_forward(handle, NULL); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_forward() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_forward() failed\n", __func__); margo_destroy(handle); return dspaces_ERR_MERCURY; } hret = margo_get_output(handle, &out); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_get_output() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_get_output() failed\n", __func__); margo_destroy(handle); return dspaces_ERR_MERCURY; } @@ -135,10 +139,12 @@ static int get_ss_info(dspaces_client_t client) { client->dcg->ss_info.num_dims = ss_data.num_dims; client->dcg->ss_info.num_space_srv = ss_data.num_space_srv; - memcpy(&(client->dcg->ss_domain), &(ss_data.ss_domain), sizeof(struct bbox)); + memcpy(&(client->dcg->ss_domain), &(ss_data.ss_domain), + sizeof(struct bbox)); client->dcg->max_versions = ss_data.max_versions; client->dcg->hash_version = ss_data.hash_version; - memcpy(&(client->dcg->default_gdim), &(ss_data.default_gdim), sizeof(struct global_dimension)); + memcpy(&(client->dcg->default_gdim), &(ss_data.default_gdim), + sizeof(struct global_dimension)); margo_free_output(handle, &out); margo_destroy(handle); @@ -146,27 +152,27 @@ static int get_ss_info(dspaces_client_t client) { return ret; } -static struct dc_gspace * dcg_alloc(dspaces_client_t client) +static struct dc_gspace *dcg_alloc(dspaces_client_t client) { - struct dc_gspace *dcg_l; - int i; + struct dc_gspace *dcg_l; + int i; - dcg_l = calloc(1, sizeof(*dcg_l)); - if (!dcg_l) - goto err_out; + dcg_l = calloc(1, sizeof(*dcg_l)); + if(!dcg_l) + goto err_out; - INIT_LIST_HEAD(&dcg_l->locks_list); - init_gdim_list(&dcg_l->gdim_list); - dcg_l->hash_version = ssd_hash_version_v1; // set default hash versio - return dcg_l; + INIT_LIST_HEAD(&dcg_l->locks_list); + init_gdim_list(&dcg_l->gdim_list); + dcg_l->hash_version = ssd_hash_version_v1; // set default hash versio + return dcg_l; - err_out: - fprintf(stderr, "'%s()': failed.\n", __func__); - return NULL; +err_out: + fprintf(stderr, "'%s()': failed.\n", __func__); + return NULL; } - -static int build_address(dspaces_client_t client){ +static int build_address(dspaces_client_t client) +{ /* open config file for reading */ int ret; struct stat st; @@ -178,74 +184,81 @@ static int build_address(dspaces_client_t client){ int wait_time, time = 0; int fd; char *file_name = "servids.0"; - + do { fd = open(file_name, O_RDONLY); - if (fd == -1) - { + if(fd == -1) { if(errno == ENOENT) { - DEBUG_OUT("unable to find config file %s after %d seconds, will try again...\n", file_name, time); + DEBUG_OUT("unable to find config file %s after %d seconds, " + "will try again...\n", + file_name, time); } else { - fprintf(stderr, "ERROR: could not open config file %s.\n", file_name); + fprintf(stderr, "ERROR: could not open config file %s.\n", + file_name); goto fini; } wait_time = (rand() % 3) + 1; time += wait_time; sleep(wait_time); } - }while(fd == -1); + } while(fd == -1); /* get file size and allocate a buffer to store it */ ret = fstat(fd, &st); - if (ret == -1) - { - fprintf(stderr, "Error: Unable to stat config file %s for server_address list\n", + if(ret == -1) { + fprintf( + stderr, + "Error: Unable to stat config file %s for server_address list\n", file_name); goto fini; } ret = -1; rd_buf = malloc(st.st_size); - if (rd_buf == NULL) goto fini; + if(rd_buf == NULL) + goto fini; /* load it all in one fell swoop */ rd_buf_size = read(fd, rd_buf, st.st_size); - if (rd_buf_size != st.st_size) - { - fprintf(stderr, "Error: Unable to stat config file %s for server_address list\n", + if(rd_buf_size != st.st_size) { + fprintf( + stderr, + "Error: Unable to stat config file %s for server_address list\n", file_name); goto fini; } - rd_buf[rd_buf_size]='\0'; + rd_buf[rd_buf_size] = '\0'; // strtok the result - each space-delimited address is assumed to be // a unique mercury address tok = strtok(rd_buf, "\r\n\t "); - if (tok == NULL) goto fini; + if(tok == NULL) + goto fini; // build up the address buffer addr_str_buf = malloc(rd_buf_size); - if (addr_str_buf == NULL) goto fini; - do - { + if(addr_str_buf == NULL) + goto fini; + do { int tok_size = strlen(tok); - memcpy((char*)addr_str_buf + addr_str_buf_len, tok, tok_size+1); - addr_str_buf_len += tok_size+1; + memcpy((char *)addr_str_buf + addr_str_buf_len, tok, tok_size + 1); + addr_str_buf_len += tok_size + 1; num_addrs++; tok = strtok(NULL, "\r\n\t "); - } while (tok != NULL); - if (addr_str_buf_len != rd_buf_size) - { + } while(tok != NULL); + if(addr_str_buf_len != rd_buf_size) { // adjust buffer size if our initial guess was wrong fprintf(stderr, "Read size and buffer_len are not equal\n"); void *tmp = realloc(addr_str_buf, addr_str_buf_len); - if (tmp == NULL) goto fini; + if(tmp == NULL) + goto fini; addr_str_buf = tmp; } free(rd_buf); - + /* set up address string array for group members */ - client->server_address = (char **)addr_str_buf_to_list(addr_str_buf, num_addrs); + client->server_address = + (char **)addr_str_buf_to_list(addr_str_buf, num_addrs); client->size_sp = num_addrs; ret = 0; @@ -259,13 +272,15 @@ static int read_conf(dspaces_client_t client, char **listen_addr_str) int size; FILE *fd; fpos_t lstart; - int i, ret; - + int i, ret; + do { fd = fopen("conf.ds", "r"); if(!fd) { if(errno == ENOENT) { - DEBUG_OUT("unable to find config file 'conf.ds' after %d seconds, will try again...\n", time); + DEBUG_OUT("unable to find config file 'conf.ds' after %d " + "seconds, will try again...\n", + time); } else { fprintf(stderr, "could not open config file 'conf.ds'.\n"); goto fini; @@ -274,10 +289,11 @@ static int read_conf(dspaces_client_t client, char **listen_addr_str) wait_time = (rand() % 3) + 1; time += wait_time; sleep(wait_time); - }while(!fd); + } while(!fd); fscanf(fd, "%d\n", &client->size_sp); - client->server_address = malloc(client->size_sp * sizeof(*client->server_address)); + client->server_address = + malloc(client->size_sp * sizeof(*client->server_address)); for(i = 0; i < client->size_sp; i++) { fgetpos(fd, &lstart); fscanf(fd, "%*s%n\n", &size); @@ -297,12 +313,13 @@ static int read_conf(dspaces_client_t client, char **listen_addr_str) return ret; } -int dspaces_init(int rank, dspaces_client_t* c) -{ +int dspaces_init(int rank, dspaces_client_t *c) +{ char *listen_addr_str; - const char *envdebug = getenv("DSPACES_DEBUG"); + const char *envdebug = getenv("DSPACES_DEBUG"); dspaces_client_t client = (dspaces_client_t)calloc(1, sizeof(*client)); - if(!client) return dspaces_ERR_ALLOCATION; + if(!client) + return dspaces_ERR_ALLOCATION; int i; if(envdebug) { @@ -311,7 +328,7 @@ int dspaces_init(int rank, dspaces_client_t* c) client->rank = rank; - //now do dcg_alloc and store gid + // now do dcg_alloc and store gid client->dcg = dcg_alloc(client); if(!(client->dcg)) @@ -345,47 +362,55 @@ int dspaces_init(int rank, dspaces_client_t* c) margo_registered_name(client->mid, "put_rpc", &id, &flag); if(flag == HG_TRUE) { /* RPCs already registered */ - margo_registered_name(client->mid, "put_rpc", &client->put_id, &flag); - margo_registered_name(client->mid, "put_local_rpc", &client->put_local_id, &flag); - margo_registered_name(client->mid, "get_rpc", &client->get_id, &flag); - margo_registered_name(client->mid, "query_rpc", &client->query_id, &flag); - margo_registered_name(client->mid, "ss_rpc", &client->ss_id, &flag); - margo_registered_name(client->mid, "drain_rpc", &client->drain_id, &flag); - margo_registered_name(client->mid, "kill_rpc", &client->kill_id, &flag); - margo_registered_name(client->mid, "sub_rpc", &client->sub_id, &flag); - margo_registered_name(client->mid, "notify_rpc", &client->notify_id, &flag); - } else { - - client->put_id = - MARGO_REGISTER(client->mid, "put_rpc", bulk_gdim_t, bulk_out_t, NULL); - client->put_local_id = - MARGO_REGISTER(client->mid, "put_local_rpc", odsc_gdim_t, bulk_out_t, NULL); - client->get_id = - MARGO_REGISTER(client->mid, "get_rpc", bulk_in_t, bulk_out_t, get_rpc); + margo_registered_name(client->mid, "put_rpc", &client->put_id, &flag); + margo_registered_name(client->mid, "put_local_rpc", + &client->put_local_id, &flag); + margo_registered_name(client->mid, "get_rpc", &client->get_id, &flag); + margo_registered_name(client->mid, "query_rpc", &client->query_id, + &flag); + margo_registered_name(client->mid, "ss_rpc", &client->ss_id, &flag); + margo_registered_name(client->mid, "drain_rpc", &client->drain_id, + &flag); + margo_registered_name(client->mid, "kill_rpc", &client->kill_id, &flag); + margo_registered_name(client->mid, "sub_rpc", &client->sub_id, &flag); + margo_registered_name(client->mid, "notify_rpc", &client->notify_id, + &flag); + } else { + + client->put_id = MARGO_REGISTER(client->mid, "put_rpc", bulk_gdim_t, + bulk_out_t, NULL); + client->put_local_id = MARGO_REGISTER(client->mid, "put_local_rpc", + odsc_gdim_t, bulk_out_t, NULL); + client->get_id = MARGO_REGISTER(client->mid, "get_rpc", bulk_in_t, + bulk_out_t, get_rpc); margo_register_data(client->mid, client->get_id, (void *)client, NULL); - client->query_id = - MARGO_REGISTER(client->mid, "query_rpc", odsc_gdim_t, odsc_list_t, NULL); + client->query_id = MARGO_REGISTER(client->mid, "query_rpc", odsc_gdim_t, + odsc_list_t, NULL); client->ss_id = MARGO_REGISTER(client->mid, "ss_rpc", void, ss_information, NULL); - client->drain_id = - MARGO_REGISTER(client->mid, "drain_rpc", bulk_in_t, bulk_out_t, drain_rpc); - margo_register_data(client->mid, client->drain_id, (void *)client, NULL); + client->drain_id = MARGO_REGISTER(client->mid, "drain_rpc", bulk_in_t, + bulk_out_t, drain_rpc); + margo_register_data(client->mid, client->drain_id, (void *)client, + NULL); client->kill_id = MARGO_REGISTER(client->mid, "kill_rpc", int32_t, void, kill_rpc); - margo_registered_disable_response(client->mid, client->kill_id, HG_TRUE); - margo_register_data(client->mid, client->kill_id, (void *)client, NULL); - client->sub_id = - MARGO_REGISTER(client->mid, "sub_rpc", odsc_gdim_t, void, NULL); - margo_registered_disable_response(client->mid, client->sub_id, HG_TRUE); - client->notify_id = - MARGO_REGISTER(client->mid, "notify_rpc", odsc_list_t, void, notify_rpc); - margo_register_data(client->mid, client->notify_id, (void *)client, NULL); - margo_registered_disable_response(client->mid, client->notify_id, HG_TRUE); - - } + margo_registered_disable_response(client->mid, client->kill_id, + HG_TRUE); + margo_register_data(client->mid, client->kill_id, (void *)client, NULL); + client->sub_id = + MARGO_REGISTER(client->mid, "sub_rpc", odsc_gdim_t, void, NULL); + margo_registered_disable_response(client->mid, client->sub_id, HG_TRUE); + client->notify_id = MARGO_REGISTER(client->mid, "notify_rpc", + odsc_list_t, void, notify_rpc); + margo_register_data(client->mid, client->notify_id, (void *)client, + NULL); + margo_registered_disable_response(client->mid, client->notify_id, + HG_TRUE); + } get_ss_info(client); - DEBUG_OUT("Total max versions on the client side is %d\n", client->dcg->max_versions); + DEBUG_OUT("Total max versions on the client side is %d\n", + client->dcg->max_versions); client->dcg->ls = ls_alloc(client->dcg->max_versions); client->local_put_count = 0; @@ -426,9 +451,10 @@ int dspaces_fini(dspaces_client_t client) client->f_final = 1; if(client->local_put_count > 0) { - DEBUG_OUT("waiting for pending drainage. %d object remain.\n", client->local_put_count); + DEBUG_OUT("waiting for pending drainage. %d object remain.\n", + client->local_put_count); ABT_cond_wait(client->drain_cond, client->drain_mutex); - DEBUG_OUT("received drainage signal.\n"); + DEBUG_OUT("received drainage signal.\n"); } ABT_mutex_unlock(client->drain_mutex); } while(client->local_put_count > 0); @@ -453,81 +479,75 @@ int dspaces_fini(dspaces_client_t client) return dspaces_SUCCESS; } -void dspaces_define_gdim (dspaces_client_t client, - const char *var_name, int ndim, uint64_t *gdim){ - if(ndim > BBOX_MAX_NDIM){ - fprintf(stderr, "ERROR: maximum object dimensionality is %d\n", BBOX_MAX_NDIM); - }else{ +void dspaces_define_gdim(dspaces_client_t client, const char *var_name, + int ndim, uint64_t *gdim) +{ + if(ndim > BBOX_MAX_NDIM) { + fprintf(stderr, "ERROR: maximum object dimensionality is %d\n", + BBOX_MAX_NDIM); + } else { update_gdim_list(&(client->dcg->gdim_list), var_name, ndim, gdim); } - - } -int dspaces_put (dspaces_client_t client, - const char *var_name, - unsigned int ver, int elem_size, - int ndim, uint64_t *lb, uint64_t *ub, - void *data) +int dspaces_put(dspaces_client_t client, const char *var_name, unsigned int ver, + int elem_size, int ndim, uint64_t *lb, uint64_t *ub, void *data) { hg_addr_t server_addr; hg_handle_t handle; - hg_return_t hret; + hg_return_t hret; int ret = dspaces_SUCCESS; - obj_descriptor odsc = { - .version = ver, .owner = {0}, - .st = st, - .size = elem_size, - .bb = {.num_dims = ndim,} - }; + obj_descriptor odsc = {.version = ver, + .owner = {0}, + .st = st, + .size = elem_size, + .bb = { + .num_dims = ndim, + }}; - memset(odsc.bb.lb.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); - memset(odsc.bb.ub.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); + memset(odsc.bb.lb.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); + memset(odsc.bb.ub.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); - memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t)*ndim); - memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t)*ndim); + memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t) * ndim); + memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t) * ndim); - strncpy(odsc.name, var_name, sizeof(odsc.name)-1); - odsc.name[sizeof(odsc.name)-1] = '\0'; + strncpy(odsc.name, var_name, sizeof(odsc.name) - 1); + odsc.name[sizeof(odsc.name) - 1] = '\0'; bulk_gdim_t in; bulk_out_t out; struct global_dimension odsc_gdim; - set_global_dimension(&(client->dcg->gdim_list), var_name, &(client->dcg->default_gdim), - &odsc_gdim); - + set_global_dimension(&(client->dcg->gdim_list), var_name, + &(client->dcg->default_gdim), &odsc_gdim); in.odsc.size = sizeof(odsc); - in.odsc.raw_odsc = (char*)(&odsc); + in.odsc.raw_odsc = (char *)(&odsc); in.odsc.gdim_size = sizeof(struct global_dimension); - in.odsc.raw_gdim = (char*)(&odsc_gdim); + in.odsc.raw_gdim = (char *)(&odsc_gdim); hg_size_t rdma_size = (elem_size)*bbox_volume(&odsc.bb); DEBUG_OUT("sending object %s \n", obj_desc_sprint(&odsc)); - hret = margo_bulk_create(client->mid, 1, (void**)&data, &rdma_size, - HG_BULK_READ_ONLY, &in.handle); + hret = margo_bulk_create(client->mid, 1, (void **)&data, &rdma_size, + HG_BULK_READ_ONLY, &in.handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_create() failed\n", __func__); return dspaces_ERR_MERCURY; } - + get_server_address(client, &server_addr); /* create handle */ - hret = margo_create( client->mid, - server_addr, - client->put_id, - &handle); + hret = margo_create(client->mid, server_addr, client->put_id, &handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); margo_bulk_free(in.handle); return dspaces_ERR_MERCURY; } hret = margo_forward(handle, &in); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_forward() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_forward() failed\n", __func__); margo_bulk_free(in.handle); margo_destroy(handle); return dspaces_ERR_MERCURY; @@ -535,7 +555,7 @@ int dspaces_put (dspaces_client_t client, hret = margo_get_output(handle, &out); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_get_output() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_get_output() failed\n", __func__); margo_bulk_free(in.handle); margo_destroy(handle); return dspaces_ERR_MERCURY; @@ -546,46 +566,43 @@ int dspaces_put (dspaces_client_t client, margo_bulk_free(in.handle); margo_destroy(handle); margo_addr_free(client->mid, server_addr); - return ret; - + return ret; } -static int get_data(dspaces_client_t client, int num_odscs, obj_descriptor req_obj, obj_descriptor *odsc_tab, void *data) +static int get_data(dspaces_client_t client, int num_odscs, + obj_descriptor req_obj, obj_descriptor *odsc_tab, + void *data) { bulk_in_t *in; - in = (bulk_in_t*)malloc(sizeof(bulk_in_t)*num_odscs); + in = (bulk_in_t *)malloc(sizeof(bulk_in_t) * num_odscs); struct obj_data **od; od = malloc(num_odscs * sizeof(struct obj_data *)); margo_request *serv_req; hg_handle_t *hndl; - hndl = (hg_handle_t*)malloc(sizeof(hg_handle_t)*num_odscs); - serv_req = (margo_request*)malloc(sizeof(margo_request)*num_odscs); + hndl = (hg_handle_t *)malloc(sizeof(hg_handle_t) * num_odscs); + serv_req = (margo_request *)malloc(sizeof(margo_request) * num_odscs); - for (int i = 0; i < num_odscs; ++i) - { + for(int i = 0; i < num_odscs; ++i) { od[i] = obj_data_alloc(&odsc_tab[i]); in[i].odsc.size = sizeof(obj_descriptor); - in[i].odsc.raw_odsc = (char*)(&odsc_tab[i]); + in[i].odsc.raw_odsc = (char *)(&odsc_tab[i]); - hg_size_t rdma_size = (req_obj.size)*bbox_volume(&odsc_tab[i].bb); + hg_size_t rdma_size = (req_obj.size) * bbox_volume(&odsc_tab[i].bb); - margo_bulk_create(client->mid, 1, (void**)(&(od[i]->data)), &rdma_size, - HG_BULK_WRITE_ONLY, &in[i].handle); + margo_bulk_create(client->mid, 1, (void **)(&(od[i]->data)), &rdma_size, + HG_BULK_WRITE_ONLY, &in[i].handle); hg_addr_t server_addr; margo_addr_lookup(client->mid, odsc_tab[i].owner, &server_addr); hg_handle_t handle; - margo_create( client->mid, - server_addr, - client->get_id, - &handle); + margo_create(client->mid, server_addr, client->get_id, &handle); margo_request req; - //forward get requests - margo_iforward(handle, &in[i], &req); + // forward get requests + margo_iforward(handle, &in[i], &req); hndl[i] = handle; serv_req[i] = req; margo_addr_free(client->mid, server_addr); @@ -593,14 +610,14 @@ static int get_data(dspaces_client_t client, int num_odscs, obj_descriptor req_o struct obj_data *return_od = obj_data_alloc_no_data(&req_obj, data); - //TODO: rewrite with margo_wait_any() - for (int i = 0; i < num_odscs; ++i){ + // TODO: rewrite with margo_wait_any() + for(int i = 0; i < num_odscs; ++i) { margo_wait(serv_req[i]); bulk_out_t resp; margo_get_output(hndl[i], &resp); margo_free_output(hndl[i], &resp); margo_destroy(hndl[i]); - //copy received data into user return buffer + // copy received data into user return buffer ssd_copy(return_od, od[i]); obj_data_free(od[i]); } @@ -608,40 +625,41 @@ static int get_data(dspaces_client_t client, int num_odscs, obj_descriptor req_o free(serv_req); free(in); free(return_od); - } static int dspaces_init_listener(dspaces_client_t client) { - + ABT_pool margo_pool; hg_return_t hret; - int ret = dspaces_SUCCESS; + int ret = dspaces_SUCCESS; hret = margo_get_handler_pool(client->mid, &margo_pool); if(hret != HG_SUCCESS || margo_pool == ABT_POOL_NULL) { - fprintf(stderr, "DSPACES_ERROR: %s: could not get handler pool (%d).\n", __func__, hret); - return(dspaces_ERR_ARGOBOTS); - } + fprintf(stderr, "DSPACES_ERROR: %s: could not get handler pool (%d).\n", + __func__, hret); + return (dspaces_ERR_ARGOBOTS); + } client->listener_xs = ABT_XSTREAM_NULL; - ret = ABT_xstream_create_basic(ABT_SCHED_BASIC_WAIT, 1, &margo_pool, ABT_SCHED_CONFIG_NULL, &client->listener_xs); - if(ret != ABT_SUCCESS) { - char err_str[1000]; - ABT_error_get_str(ret, err_str, NULL); - fprintf(stderr, "DSPACES ERROR: %s: could not launch handler thread: %s\n", __func__, err_str); - return(dspaces_ERR_ARGOBOTS); - } + ret = ABT_xstream_create_basic(ABT_SCHED_BASIC_WAIT, 1, &margo_pool, + ABT_SCHED_CONFIG_NULL, &client->listener_xs); + if(ret != ABT_SUCCESS) { + char err_str[1000]; + ABT_error_get_str(ret, err_str, NULL); + fprintf(stderr, + "DSPACES ERROR: %s: could not launch handler thread: %s\n", + __func__, err_str); + return (dspaces_ERR_ARGOBOTS); + } client->listener_init = 1; - return(ret); + return (ret); } -int dspaces_put_local (dspaces_client_t client, - const char *var_name, - unsigned int ver, int elem_size, - int ndim, uint64_t *lb, uint64_t *ub, - void *data) +int dspaces_put_local(dspaces_client_t client, const char *var_name, + unsigned int ver, int elem_size, int ndim, uint64_t *lb, + uint64_t *ub, void *data) { hg_addr_t server_addr; hg_handle_t handle; @@ -649,20 +667,20 @@ int dspaces_put_local (dspaces_client_t client, int ret = dspaces_SUCCESS; if(client->listener_init == 0) { - ret = dspaces_init_listener(client); - if(ret != dspaces_SUCCESS) { - return(ret); - } + ret = dspaces_init_listener(client); + if(ret != dspaces_SUCCESS) { + return (ret); + } } - - client->local_put_count++; - obj_descriptor odsc = { - .version = ver, - .st = st, - .size = elem_size, - .bb = {.num_dims = ndim,} - }; + client->local_put_count++; + + obj_descriptor odsc = {.version = ver, + .st = st, + .size = elem_size, + .bb = { + .num_dims = ndim, + }}; hg_addr_t owner_addr; size_t owner_addr_size = 128; @@ -671,57 +689,54 @@ int dspaces_put_local (dspaces_client_t client, margo_addr_to_string(client->mid, odsc.owner, &owner_addr_size, owner_addr); margo_addr_free(client->mid, owner_addr); - memset(odsc.bb.lb.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); - memset(odsc.bb.ub.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); + memset(odsc.bb.lb.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); + memset(odsc.bb.ub.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); - memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t)*ndim); - memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t)*ndim); + memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t) * ndim); + memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t) * ndim); - strncpy(odsc.name, var_name, sizeof(odsc.name)-1); - odsc.name[sizeof(odsc.name)-1] = '\0'; + strncpy(odsc.name, var_name, sizeof(odsc.name) - 1); + odsc.name[sizeof(odsc.name) - 1] = '\0'; odsc_gdim_t in; bulk_out_t out; struct obj_data *od; od = obj_data_alloc_with_data(&odsc, data); - set_global_dimension(&(client->dcg->gdim_list), var_name, &(client->dcg->default_gdim), - &od->gdim); + set_global_dimension(&(client->dcg->gdim_list), var_name, + &(client->dcg->default_gdim), &od->gdim); - ABT_mutex_lock(client->ls_mutex); ls_add_obj(client->dcg->ls, od); DEBUG_OUT("Added into local_storage\n"); ABT_mutex_unlock(client->ls_mutex); in.odsc_gdim.size = sizeof(odsc); - in.odsc_gdim.raw_odsc = (char*)(&odsc); + in.odsc_gdim.raw_odsc = (char *)(&odsc); in.odsc_gdim.gdim_size = sizeof(struct global_dimension); - in.odsc_gdim.raw_gdim = (char*)(&od->gdim); + in.odsc_gdim.raw_gdim = (char *)(&od->gdim); DEBUG_OUT("sending object information %s \n", obj_desc_sprint(&odsc)); - + get_server_address(client, &server_addr); /* create handle */ - hret = margo_create( client->mid, - server_addr, - client->put_local_id, - &handle); + hret = + margo_create(client->mid, server_addr, client->put_local_id, &handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); return dspaces_ERR_MERCURY; } hret = margo_forward(handle, &in); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_forward() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_forward() failed\n", __func__); margo_destroy(handle); return dspaces_ERR_MERCURY; } hret = margo_get_output(handle, &out); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_get_output() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_get_output() failed\n", __func__); margo_destroy(handle); return dspaces_ERR_MERCURY; } @@ -732,57 +747,52 @@ int dspaces_put_local (dspaces_client_t client, margo_addr_free(client->mid, server_addr); return ret; - } -int dspaces_get (dspaces_client_t client, - const char *var_name, - unsigned int ver, int elem_size, - int ndim, uint64_t *lb, uint64_t *ub, - void *data, int timeout) +int dspaces_get(dspaces_client_t client, const char *var_name, unsigned int ver, + int elem_size, int ndim, uint64_t *lb, uint64_t *ub, void *data, + int timeout) { hg_addr_t server_addr; hg_handle_t handle; hg_return_t hret; int ret = dspaces_SUCCESS; - obj_descriptor odsc = { - .version = ver, .owner = {0}, - .st = st, - .size = elem_size, - .bb = {.num_dims = ndim,} - }; + obj_descriptor odsc = {.version = ver, + .owner = {0}, + .st = st, + .size = elem_size, + .bb = { + .num_dims = ndim, + }}; - memset(odsc.bb.lb.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); - memset(odsc.bb.ub.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); + memset(odsc.bb.lb.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); + memset(odsc.bb.ub.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); - memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t)*ndim); - memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t)*ndim); + memcpy(odsc.bb.lb.c, lb, sizeof(uint64_t) * ndim); + memcpy(odsc.bb.ub.c, ub, sizeof(uint64_t) * ndim); - strncpy(odsc.name, var_name, sizeof(odsc.name)-1); - odsc.name[sizeof(odsc.name)-1] = '\0'; + strncpy(odsc.name, var_name, sizeof(odsc.name) - 1); + odsc.name[sizeof(odsc.name) - 1] = '\0'; odsc_gdim_t in; odsc_list_t out; in.odsc_gdim.size = sizeof(odsc); - in.odsc_gdim.raw_odsc = (char*)(&odsc); + in.odsc_gdim.raw_odsc = (char *)(&odsc); in.param = timeout; struct global_dimension od_gdim; - set_global_dimension(&(client->dcg->gdim_list), var_name, &(client->dcg->default_gdim), - &od_gdim); + set_global_dimension(&(client->dcg->gdim_list), var_name, + &(client->dcg->default_gdim), &od_gdim); in.odsc_gdim.gdim_size = sizeof(struct global_dimension); - in.odsc_gdim.raw_gdim = (char*)(&od_gdim); + in.odsc_gdim.raw_gdim = (char *)(&od_gdim); get_server_address(client, &server_addr); - hret = margo_create( client->mid, - server_addr, - client->query_id, - &handle); + hret = margo_create(client->mid, server_addr, client->query_id, &handle); assert(hret == HG_SUCCESS); hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS); @@ -791,24 +801,22 @@ int dspaces_get (dspaces_client_t client, assert(hret == HG_SUCCESS); obj_descriptor *odsc_tab; - int num_odscs = (out.odsc_list.size)/sizeof(obj_descriptor); + int num_odscs = (out.odsc_list.size) / sizeof(obj_descriptor); odsc_tab = malloc(out.odsc_list.size); memcpy(odsc_tab, out.odsc_list.raw_odsc, out.odsc_list.size); margo_free_output(handle, &out); margo_destroy(handle); - + DEBUG_OUT("Finished query - need to fetch %d objects\n", num_odscs); - for (int i = 0; i < num_odscs; ++i) - { + for(int i = 0; i < num_odscs; ++i) { DEBUG_OUT("%s\n", obj_desc_sprint(&odsc_tab[i])); } - //send request to get the obj_desc - if(num_odscs!=0) + // send request to get the obj_desc + if(num_odscs != 0) get_data(client, num_odscs, odsc, odsc_tab, data); margo_addr_free(client->mid, server_addr); - } static void get_rpc(hg_handle_t handle) @@ -820,43 +828,46 @@ static void get_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_client_t client = (dspaces_client_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_client_t client = + (dspaces_client_t)margo_registered_data(mid, info->id); DEBUG_OUT("Received rpc to get data\n"); hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + assert(hret == HG_SUCCESS); obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc.raw_odsc, sizeof(in_odsc)); DEBUG_OUT("%s\n", obj_desc_sprint(&in_odsc)); - + struct obj_data *od, *from_obj; from_obj = ls_find(client->dcg->ls, &in_odsc); if(!from_obj) - fprintf(stderr, "WARNING: (%s): Object not found in local storage\n", __func__); + fprintf(stderr, "WARNING: (%s): Object not found in local storage\n", + __func__); od = obj_data_alloc(&in_odsc); if(!od) fprintf(stderr, "ERROR: (%s): object allocation failed\n", __func__); if(from_obj->data == NULL) - fprintf(stderr, "ERROR: (%s): object data allocation failed\n", __func__); + fprintf(stderr, "ERROR: (%s): object data allocation failed\n", + __func__); ssd_copy(od, from_obj); DEBUG_OUT("After ssd_copy\n"); - - hg_size_t size = (in_odsc.size)*bbox_volume(&(in_odsc.bb)); - void *buffer = (void*) od->data; - hret = margo_bulk_create(mid, 1, (void**)&buffer, &size, - HG_BULK_READ_ONLY, &bulk_handle); + hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); + void *buffer = (void *)od->data; + + hret = margo_bulk_create(mid, 1, (void **)&buffer, &size, HG_BULK_READ_ONLY, + &bulk_handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_create() failed\n", __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); @@ -865,9 +876,10 @@ static void get_rpc(hg_handle_t handle) } hret = margo_bulk_transfer(mid, HG_BULK_PUSH, info->addr, in.handle, 0, - bulk_handle, 0, size); + bulk_handle, 0, size); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_transfer() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_transfer() failed\n", + __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); @@ -893,38 +905,40 @@ static void drain_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_client_t client = (dspaces_client_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_client_t client = + (dspaces_client_t)margo_registered_data(mid, info->id); DEBUG_OUT("Received rpc to drain data\n"); hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + assert(hret == HG_SUCCESS); obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc.raw_odsc, sizeof(in_odsc)); DEBUG_OUT("%s\n", obj_desc_sprint(&in_odsc)); - + struct obj_data *from_obj; from_obj = ls_find(client->dcg->ls, &in_odsc); - if(!from_obj){ - fprintf(stderr, "Object not found in client's local storage.\n Make sure MAX version is set appropriately in dataspaces.conf\n"); + if(!from_obj) { + fprintf(stderr, + "Object not found in client's local storage.\n Make sure MAX " + "version is set appropriately in dataspaces.conf\n"); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); - return; + return; } - - hg_size_t size = (in_odsc.size)*bbox_volume(&(in_odsc.bb)); - void *buffer = (void*) from_obj->data; + hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); + void *buffer = (void *)from_obj->data; - hret = margo_bulk_create(mid, 1, (void**)&buffer, &size, - HG_BULK_READ_ONLY, &bulk_handle); + hret = margo_bulk_create(mid, 1, (void **)&buffer, &size, HG_BULK_READ_ONLY, + &bulk_handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_create() failed\n", __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); @@ -933,9 +947,10 @@ static void drain_rpc(hg_handle_t handle) } hret = margo_bulk_transfer(mid, HG_BULK_PUSH, info->addr, in.handle, 0, - bulk_handle, 0, size); + bulk_handle, 0, size); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_transfer() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_transfer() failed\n", + __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); @@ -944,14 +959,13 @@ static void drain_rpc(hg_handle_t handle) return; } margo_bulk_free(bulk_handle); - out.ret = dspaces_SUCCESS; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); - //delete object from local storage - DEBUG_OUT("Finished draining %s\n", obj_desc_sprint(&from_obj->obj_desc)); + // delete object from local storage + DEBUG_OUT("Finished draining %s\n", obj_desc_sprint(&from_obj->obj_desc)); ABT_mutex_lock(client->ls_mutex); ls_try_remove_free(client->dcg->ls, from_obj); ABT_mutex_unlock(client->ls_mutex); @@ -967,7 +981,8 @@ static void drain_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(drain_rpc) -static struct dspaces_sub_handle *dspaces_get_sub(dspaces_client_t client, int sub_id) +static struct dspaces_sub_handle *dspaces_get_sub(dspaces_client_t client, + int sub_id) { int listidx = sub_id % SUB_HASH_SIZE; struct sub_list_node *node, **nodep; @@ -975,12 +990,15 @@ static struct dspaces_sub_handle *dspaces_get_sub(dspaces_client_t client, int s node = client->sub_lists[listidx]; while(node) { if(node->id == sub_id) { - return(node->subh); + return (node->subh); } } - fprintf(stderr, "WARNING: received notification for unknown subscription id %d. This shouldn't happen.\n", sub_id); - return(NULL); + fprintf(stderr, + "WARNING: received notification for unknown subscription id %d. " + "This shouldn't happen.\n", + sub_id); + return (NULL); } static void dspaces_move_sub(dspaces_client_t client, int sub_id) @@ -995,7 +1013,10 @@ static void dspaces_move_sub(dspaces_client_t client, int sub_id) } if(!*nodep) { - fprintf(stderr, "WARNING: trying to mark unknown sub %d done. This shouldn't happen.\n", sub_id); + fprintf(stderr, + "WARNING: trying to mark unknown sub %d done. This shouldn't " + "happen.\n", + sub_id); return; } @@ -1020,8 +1041,9 @@ static void free_sub_req(struct dspaces_req *req) static void notify_rpc(hg_handle_t handle) { margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_client_t client = (dspaces_client_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_client_t client = + (dspaces_client_t)margo_registered_data(mid, info->id); odsc_list_t in; struct dspaces_sub_handle *subh; int sub_id; @@ -1041,18 +1063,20 @@ static void notify_rpc(hg_handle_t handle) if(subh->status == DSPACES_SUB_WAIT) { ABT_mutex_unlock(client->sub_mutex); - num_odscs = (in.odsc_list.size)/sizeof(obj_descriptor); + num_odscs = (in.odsc_list.size) / sizeof(obj_descriptor); odsc_tab = malloc(in.odsc_list.size); memcpy(odsc_tab, in.odsc_list.raw_odsc, in.odsc_list.size); - DEBUG_OUT("Satisfying subscription requires fetching %d objects:\n", num_odscs); + DEBUG_OUT("Satisfying subscription requires fetching %d objects:\n", + num_odscs); for(i = 0; i < num_odscs; i++) { DEBUG_OUT("%s\n", obj_desc_sprint(&odsc_tab[i])); } data_size = subh->q_odsc.size; for(i = 0; i < subh->q_odsc.bb.num_dims; i++) { - data_size *= (subh->q_odsc.bb.ub.c[i] - subh->q_odsc.bb.lb.c[i]) + 1; + data_size *= + (subh->q_odsc.bb.ub.c[i] - subh->q_odsc.bb.lb.c[i]) + 1; } data = malloc(data_size); @@ -1081,7 +1105,7 @@ static void notify_rpc(hg_handle_t handle) if(data) { subh->result = subh->cb(client, subh->req, subh->arg); - } + } ABT_mutex_lock(client->sub_mutex); client->pending_sub--; @@ -1097,7 +1121,8 @@ static void notify_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(notify_rpc) -static void register_client_sub(dspaces_client_t client, struct dspaces_sub_handle *subh) +static void register_client_sub(dspaces_client_t client, + struct dspaces_sub_handle *subh) { int listidx = subh->id % SUB_HASH_SIZE; struct sub_list_node **node = &client->sub_lists[listidx]; @@ -1113,29 +1138,29 @@ static void register_client_sub(dspaces_client_t client, struct dspaces_sub_hand } struct dspaces_sub_handle *dspaces_sub(dspaces_client_t client, - const char *var_name, - unsigned int ver, int elem_size, - int ndim, uint64_t *lb, uint64_t *ub, - dspaces_sub_fn sub_cb, void *arg) + const char *var_name, unsigned int ver, + int elem_size, int ndim, uint64_t *lb, + uint64_t *ub, dspaces_sub_fn sub_cb, + void *arg) { hg_addr_t my_addr, server_addr; hg_handle_t handle; hg_return_t hret; struct dspaces_sub_handle *subh; - odsc_gdim_t in; + odsc_gdim_t in; struct global_dimension od_gdim; size_t owner_addr_size = 128; int ret; if(client->listener_init == 0) { - ret = dspaces_init_listener(client); + ret = dspaces_init_listener(client); if(ret != dspaces_SUCCESS) { - return(DSPACES_SUB_FAIL); + return (DSPACES_SUB_FAIL); } } subh = malloc(sizeof(*subh)); - + subh->req = malloc(sizeof(*subh->req)); subh->req->var_name = strdup(var_name); subh->req->ver = ver; @@ -1153,35 +1178,38 @@ struct dspaces_sub_handle *dspaces_sub(dspaces_client_t client, subh->arg = arg; subh->cb = sub_cb; - + ABT_mutex_lock(client->sub_mutex); client->pending_sub++; subh->id = client->sub_serial++; register_client_sub(client, subh); ABT_mutex_unlock(client->sub_mutex); - - memset(subh->q_odsc.bb.lb.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); - memset(subh->q_odsc.bb.ub.c, 0, sizeof(uint64_t)*BBOX_MAX_NDIM); - memcpy(subh->q_odsc.bb.lb.c, lb, sizeof(uint64_t)*ndim); - memcpy(subh->q_odsc.bb.ub.c, ub, sizeof(uint64_t)*ndim); + memset(subh->q_odsc.bb.lb.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); + memset(subh->q_odsc.bb.ub.c, 0, sizeof(uint64_t) * BBOX_MAX_NDIM); + memcpy(subh->q_odsc.bb.lb.c, lb, sizeof(uint64_t) * ndim); + memcpy(subh->q_odsc.bb.ub.c, ub, sizeof(uint64_t) * ndim); strncpy(subh->q_odsc.name, var_name, strlen(var_name) + 1); - // A hack to send our address to the server without using more space. This field is ignored in a normal query. + // A hack to send our address to the server without using more space. This + // field is ignored in a normal query. margo_addr_self(client->mid, &my_addr); - margo_addr_to_string(client->mid, subh->q_odsc.owner, &owner_addr_size, my_addr); + margo_addr_to_string(client->mid, subh->q_odsc.owner, &owner_addr_size, + my_addr); margo_addr_free(client->mid, my_addr); in.odsc_gdim.size = sizeof(subh->q_odsc); in.odsc_gdim.raw_odsc = (char *)(&subh->q_odsc); in.param = subh->id; - - DEBUG_OUT("registered data subscription for %s with id %d\n", obj_desc_sprint(&subh->q_odsc), subh->id); - - set_global_dimension(&(client->dcg->gdim_list), var_name, &(client->dcg->default_gdim), &od_gdim); + + DEBUG_OUT("registered data subscription for %s with id %d\n", + obj_desc_sprint(&subh->q_odsc), subh->id); + + set_global_dimension(&(client->dcg->gdim_list), var_name, + &(client->dcg->default_gdim), &od_gdim); in.odsc_gdim.gdim_size = sizeof(struct global_dimension); in.odsc_gdim.raw_gdim = (char *)(&od_gdim); - + get_server_address(client, &server_addr); hret = margo_create(client->mid, server_addr, client->sub_id, &handle); @@ -1189,16 +1217,18 @@ struct dspaces_sub_handle *dspaces_sub(dspaces_client_t client, hret = margo_forward(handle, &in); assert(hret == HG_SUCCESS && "margo_forward succeeds"); - DEBUG_OUT("subscription %d sent.\n", subh->id); + DEBUG_OUT("subscription %d sent.\n", subh->id); subh->status = DSPACES_SUB_WAIT; - return(subh); + return (subh); } -int dspaces_check_sub(dspaces_client_t client, dspaces_sub_t subh, int wait, int *result) +int dspaces_check_sub(dspaces_client_t client, dspaces_sub_t subh, int wait, + int *result) { if(subh == DSPACES_SUB_FAIL) { - fprintf(stderr, "WARNING: status check on invalid subscription handle.\n"); + fprintf(stderr, + "WARNING: status check on invalid subscription handle.\n"); return DSPACES_SUB_INVALID; } @@ -1207,9 +1237,10 @@ int dspaces_check_sub(dspaces_client_t client, dspaces_sub_t subh, int wait, int if(wait) { DEBUG_OUT("blocking on notification for subscription %d.\n", subh->id); ABT_mutex_lock(client->sub_mutex); - while(subh->status == DSPACES_SUB_WAIT || subh->status == DSPACES_SUB_RUNNING) { + while(subh->status == DSPACES_SUB_WAIT || + subh->status == DSPACES_SUB_RUNNING) { ABT_cond_wait(client->sub_cond, client->sub_mutex); - } + } ABT_mutex_unlock(client->sub_mutex); } @@ -1217,14 +1248,15 @@ int dspaces_check_sub(dspaces_client_t client, dspaces_sub_t subh, int wait, int *result = subh->result; } - return(subh->status); + return (subh->status); } static void kill_rpc(hg_handle_t handle) { margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_client_t client = (dspaces_client_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_client_t client = + (dspaces_client_t)margo_registered_data(mid, info->id); DEBUG_OUT("Received kill message.\n"); @@ -1240,15 +1272,15 @@ DEFINE_MARGO_RPC_HANDLER(kill_rpc) int dspaces_cancel_sub(dspaces_client_t client, dspaces_sub_t subh) { if(subh == DSPACES_SUB_FAIL) { - return(DSPACES_SUB_INVALID); + return (DSPACES_SUB_INVALID); } ABT_mutex_lock(client->sub_mutex); if(subh->status == DSPACES_SUB_WAIT) { subh->status = DSPACES_SUB_CANCELLED; } - ABT_mutex_unlock(client->sub_mutex); + ABT_mutex_unlock(client->sub_mutex); - return(0); + return (0); } void dspaces_kill(dspaces_client_t client) @@ -1265,7 +1297,7 @@ void dspaces_kill(dspaces_client_t client) margo_addr_lookup(client->mid, client->server_address[0], &server_addr); hret = margo_create(client->mid, server_addr, client->kill_id, &h); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_create() failed\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_create() failed\n", __func__); margo_addr_free(client->mid, server_addr); return; } diff --git a/src/dspaces-server.c b/src/dspaces-server.c index 83b6b43c..d3975834 100644 --- a/src/dspaces-server.c +++ b/src/dspaces-server.c @@ -1,37 +1,36 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "dspaces-server.h" +#include "gspace.h" +#include "ss_data.h" +#include +#include +#include #include #include #include -#include -#include #include -#include -#include -#include -#include "ss_data.h" -#include "dspaces-server.h" -#include "gspace.h" +#include +#include -#define DEBUG_OUT(args...) \ - do { \ - if(server->f_debug) { \ - fprintf(stderr, "Rank %i: %s, line %i (%s): ", server->rank, __FILE__, __LINE__, __func__); \ - fprintf(stderr, args); \ - } \ - }while(0); +#define DEBUG_OUT(args...) \ + do { \ + if(server->f_debug) { \ + fprintf(stderr, "Rank %i: %s, line %i (%s): ", server->rank, \ + __FILE__, __LINE__, __func__); \ + fprintf(stderr, args); \ + } \ + } while(0); #define DSPACES_DEFAULT_NUM_HANDLERS 4 static enum storage_type st = column_major; -typedef enum obj_update_type { - DS_OBJ_NEW, - DS_OBJ_OWNER -} obj_update_t; +typedef enum obj_update_type { DS_OBJ_NEW, DS_OBJ_OWNER } obj_update_t; int cond_num = 0; @@ -40,21 +39,21 @@ struct addr_list_entry { char *addr; }; -struct dspaces_provider{ +struct dspaces_provider { margo_instance_id mid; hg_id_t put_id; hg_id_t put_local_id; hg_id_t query_id; hg_id_t get_id; - hg_id_t obj_update_id; + hg_id_t obj_update_id; hg_id_t odsc_internal_id; hg_id_t ss_id; hg_id_t drain_id; hg_id_t kill_id; hg_id_t sub_id; hg_id_t notify_id; - struct ds_gspace *dsg; - char **server_address; + struct ds_gspace *dsg; + char **server_address; char *listen_addr_str; int rank; int comm_size; @@ -75,7 +74,6 @@ struct dspaces_provider{ ABT_thread drain_t; }; - DECLARE_MARGO_RPC_HANDLER(put_rpc); DECLARE_MARGO_RPC_HANDLER(put_local_rpc); DECLARE_MARGO_RPC_HANDLER(get_rpc); @@ -95,146 +93,145 @@ static void odsc_internal_rpc(hg_handle_t h); static void ss_rpc(hg_handle_t h); static void kill_rpc(hg_handle_t h); static void sub_rpc(hg_handle_t h); -//static void write_lock_rpc(hg_handle_t h); -//static void read_lock_rpc(hg_handle_t h); - - +// static void write_lock_rpc(hg_handle_t h); +// static void read_lock_rpc(hg_handle_t h); /* Server configuration parameters */ static struct { - int ndim; - struct coord dims; - int max_versions; - int max_readers; - int lock_type; /* 1 - generic, 2 - custom */ - int hash_version; /* 1 - ssd_hash_version_v1, 2 - ssd_hash_version_v2 */ - int num_apps; + int ndim; + struct coord dims; + int max_versions; + int max_readers; + int lock_type; /* 1 - generic, 2 - custom */ + int hash_version; /* 1 - ssd_hash_version_v1, 2 - ssd_hash_version_v2 */ + int num_apps; } ds_conf; static struct { - const char *opt; - int *pval; -} options[] = { - {"ndim", &ds_conf.ndim}, - {"dims", (int *)&ds_conf.dims}, - {"max_versions", &ds_conf.max_versions}, - {"max_readers", &ds_conf.max_readers}, - {"lock_type", &ds_conf.lock_type}, - {"hash_version", &ds_conf.hash_version}, - {"num_apps", &ds_conf.num_apps} -}; + const char *opt; + int *pval; +} options[] = {{"ndim", &ds_conf.ndim}, + {"dims", (int *)&ds_conf.dims}, + {"max_versions", &ds_conf.max_versions}, + {"max_readers", &ds_conf.max_readers}, + {"lock_type", &ds_conf.lock_type}, + {"hash_version", &ds_conf.hash_version}, + {"num_apps", &ds_conf.num_apps}}; static void eat_spaces(char *line) { - char *t = line; + char *t = line; - while (t && *t) { - if (*t != ' ' && *t != '\t' && *t != '\n') - *line++ = *t; - t++; - } - if (line) - *line = '\0'; + while(t && *t) { + if(*t != ' ' && *t != '\t' && *t != '\n') + *line++ = *t; + t++; + } + if(line) + *line = '\0'; } static int parse_line(int lineno, char *line) { - char *t; - int i, n; - - /* Comment line ? */ - if (line[0] == '#') - return 0; - - t = strstr(line, "="); - if (!t) { - eat_spaces(line); - if (strlen(line) == 0) - return 0; - else return -EINVAL; - } + char *t; + int i, n; - t[0] = '\0'; + /* Comment line ? */ + if(line[0] == '#') + return 0; + + t = strstr(line, "="); + if(!t) { eat_spaces(line); - t++; + if(strlen(line) == 0) + return 0; + else + return -EINVAL; + } - n = sizeof(options) / sizeof(options[0]); - - for (i = 0; i < n; i++) { - if (strcmp(line, options[1].opt) == 0){ /**< when "dims" */ - //get coordinates - int idx = 0; - char* crd; - crd = strtok(t, ","); - while(crd != NULL){ - ((struct coord*)options[1].pval)->c[idx] = atoll(crd); - crd = strtok(NULL, ","); - idx++; - } - if(idx != *(int*)options[0].pval){ - fprintf(stderr, "ERROR: (%s): dimensionality mismatch.\n", __func__); - fprintf(stderr, "ERROR: index=%d, ndims=%d\n",idx, *(int*)options[0].pval); - return -EINVAL; - } - break; - } - if (strcmp(line, options[i].opt) == 0) { - eat_spaces(line); - *(int*)options[i].pval = atoi(t); - break; - } + t[0] = '\0'; + eat_spaces(line); + t++; + + n = sizeof(options) / sizeof(options[0]); + + for(i = 0; i < n; i++) { + if(strcmp(line, options[1].opt) == 0) { /**< when "dims" */ + // get coordinates + int idx = 0; + char *crd; + crd = strtok(t, ","); + while(crd != NULL) { + ((struct coord *)options[1].pval)->c[idx] = atoll(crd); + crd = strtok(NULL, ","); + idx++; + } + if(idx != *(int *)options[0].pval) { + fprintf(stderr, "ERROR: (%s): dimensionality mismatch.\n", + __func__); + fprintf(stderr, "ERROR: index=%d, ndims=%d\n", idx, + *(int *)options[0].pval); + return -EINVAL; + } + break; } - - if (i == n) { - fprintf(stderr, "WARNING: (%s): unknown option '%s' at line %d.\n", __func__, line, lineno); + if(strcmp(line, options[i].opt) == 0) { + eat_spaces(line); + *(int *)options[i].pval = atoi(t); + break; } - return 0; + } + + if(i == n) { + fprintf(stderr, "WARNING: (%s): unknown option '%s' at line %d.\n", + __func__, line, lineno); + } + return 0; } static int parse_conf(char *fname) { - FILE *fin; - char buff[1024]; - int lineno = 1, err; - - fin = fopen(fname, "rt"); - if (!fin) - return -errno; - - while (fgets(buff, sizeof(buff), fin) != NULL) { - err = parse_line(lineno++, buff); - if (err < 0) { - fclose(fin); - return err; - } + FILE *fin; + char buff[1024]; + int lineno = 1, err; + + fin = fopen(fname, "rt"); + if(!fin) + return -errno; + + while(fgets(buff, sizeof(buff), fin) != NULL) { + err = parse_line(lineno++, buff); + if(err < 0) { + fclose(fin); + return err; } + } - fclose(fin); - return 0; + fclose(fin); + return 0; } - static int init_sspace(struct bbox *default_domain, struct ds_gspace *dsg_l) { int err = -ENOMEM; - dsg_l->ssd = ssd_alloc(default_domain, dsg_l->size_sp, - ds_conf.max_versions, ds_conf.hash_version); - if (!dsg_l->ssd) + dsg_l->ssd = ssd_alloc(default_domain, dsg_l->size_sp, ds_conf.max_versions, + ds_conf.hash_version); + if(!dsg_l->ssd) goto err_out; err = ssd_init(dsg_l->ssd, dsg_l->rank); - if (err < 0) + if(err < 0) goto err_out; dsg_l->default_gdim.ndim = ds_conf.ndim; int i; - for (i = 0; i < ds_conf.ndim; i++) { + for(i = 0; i < ds_conf.ndim; i++) { dsg_l->default_gdim.sizes.c[i] = ds_conf.dims.c[i]; } INIT_LIST_HEAD(&dsg_l->sspace_list); return 0; - err_out: +err_out: fprintf(stderr, "%s(): ERROR failed\n", __func__); return err; } @@ -254,22 +251,26 @@ static int write_conf(dspaces_provider_t server, MPI_Comm comm) hret = margo_addr_self(server->mid, &my_addr); if(hret != HG_SUCCESS) { - fprintf(stderr, "ERROR: (%s): margo_addr_self() returned %d\n", __func__, hret); + fprintf(stderr, "ERROR: (%s): margo_addr_self() returned %d\n", + __func__, hret); ret = -1; goto error; - } + } hret = margo_addr_to_string(server->mid, NULL, &my_addr_size, my_addr); if(hret != HG_SUCCESS) { - fprintf(stderr, "ERROR: (%s): margo_addr_to_string() returned %d\n", __func__, hret); + fprintf(stderr, "ERROR: (%s): margo_addr_to_string() returned %d\n", + __func__, hret); ret = -1; goto errorfree; } my_addr_str = malloc(my_addr_size); - hret = margo_addr_to_string(server->mid, my_addr_str, &my_addr_size, my_addr); + hret = + margo_addr_to_string(server->mid, my_addr_str, &my_addr_size, my_addr); if(hret != HG_SUCCESS) { - fprintf(stderr, "ERROR: (%s): margo_addr_to_string() returned %d\n", __func__, hret); + fprintf(stderr, "ERROR: (%s): margo_addr_to_string() returned %d\n", + __func__, hret); ret = -1; goto errorfree; } @@ -282,13 +283,15 @@ static int write_conf(dspaces_provider_t server, MPI_Comm comm) for(i = 0; i < server->comm_size; i++) { addr_buf_size += addr_sizes[i]; if(i) { - sizes_psum[i] = sizes_psum[i-1] + addr_sizes[i-1]; + sizes_psum[i] = sizes_psum[i - 1] + addr_sizes[i - 1]; } - } + } addr_str_buf = malloc(addr_buf_size); - MPI_Allgatherv(my_addr_str, my_addr_size, MPI_CHAR, addr_str_buf, addr_sizes, sizes_psum, MPI_CHAR, comm); + MPI_Allgatherv(my_addr_str, my_addr_size, MPI_CHAR, addr_str_buf, + addr_sizes, sizes_psum, MPI_CHAR, comm); - server->server_address = malloc(server->comm_size * sizeof(*server->server_address)); + server->server_address = + malloc(server->comm_size * sizeof(*server->server_address)); for(i = 0; i < server->comm_size; i++) { server->server_address[i] = &addr_str_buf[sizes_psum[i]]; } @@ -297,9 +300,11 @@ static int write_conf(dspaces_provider_t server, MPI_Comm comm) if(server->rank == 0) { fd = fopen("conf.ds", "w"); if(!fd) { - fprintf(stderr, "ERROR: %s: unable to open 'conf.ds' for writing.\n", __func__); - ret = -1; - goto errorfree; + fprintf(stderr, + "ERROR: %s: unable to open 'conf.ds' for writing.\n", + __func__); + ret = -1; + goto errorfree; } fprintf(fd, "%d\n", server->comm_size); for(i = 0; i < server->comm_size; i++) { @@ -309,95 +314,99 @@ static int write_conf(dspaces_provider_t server, MPI_Comm comm) fclose(fd); } - free(my_addr_str); + free(my_addr_str); free(addr_sizes); free(sizes_psum); margo_addr_free(server->mid, my_addr); - return(ret); + return (ret); errorfree: margo_addr_free(server->mid, my_addr); error: margo_finalize(server->mid); - return(ret); + return (ret); } static int dsg_alloc(dspaces_provider_t server, char *conf_name, MPI_Comm comm) { - struct ds_gspace *dsg_l; - int err = -ENOMEM; - - /* Default values */ - ds_conf.max_versions = 1; - ds_conf.max_readers = 1; - ds_conf.lock_type = 1; - ds_conf.hash_version = ssd_hash_version_v1; - ds_conf.num_apps = 1; - - err = parse_conf(conf_name); - if (err < 0) { - fprintf(stderr, "%s(): ERROR failed to load config file '%s'.", __func__, conf_name); - goto err_out; - } + struct ds_gspace *dsg_l; + int err = -ENOMEM; - // Check number of dimension - if (ds_conf.ndim > BBOX_MAX_NDIM) { - fprintf(stderr, "%s(): ERROR maximum number of array dimension is %d but ndim is %d" - " in file '%s'\n", __func__, BBOX_MAX_NDIM, ds_conf.ndim, conf_name); - err = -EINVAL; - goto err_out; - } + /* Default values */ + ds_conf.max_versions = 1; + ds_conf.max_readers = 1; + ds_conf.lock_type = 1; + ds_conf.hash_version = ssd_hash_version_v1; + ds_conf.num_apps = 1; + + err = parse_conf(conf_name); + if(err < 0) { + fprintf(stderr, "%s(): ERROR failed to load config file '%s'.", + __func__, conf_name); + goto err_out; + } + + // Check number of dimension + if(ds_conf.ndim > BBOX_MAX_NDIM) { + fprintf( + stderr, + "%s(): ERROR maximum number of array dimension is %d but ndim is %d" + " in file '%s'\n", + __func__, BBOX_MAX_NDIM, ds_conf.ndim, conf_name); + err = -EINVAL; + goto err_out; + } - // Check hash version - if ((ds_conf.hash_version < ssd_hash_version_v1) || - (ds_conf.hash_version >= _ssd_hash_version_count)) { - fprintf(stderr, "%s(): ERROR unknown hash version %d in file '%s'\n", + // Check hash version + if((ds_conf.hash_version < ssd_hash_version_v1) || + (ds_conf.hash_version >= _ssd_hash_version_count)) { + fprintf(stderr, "%s(): ERROR unknown hash version %d in file '%s'\n", __func__, ds_conf.hash_version, conf_name); - err = -EINVAL; - goto err_out; - } + err = -EINVAL; + goto err_out; + } - struct bbox domain; - memset(&domain, 0, sizeof(struct bbox)); - domain.num_dims = ds_conf.ndim; - int i; - for(i = 0; i < domain.num_dims; i++){ - domain.lb.c[i] = 0; - domain.ub.c[i] = ds_conf.dims.c[i] - 1; - } + struct bbox domain; + memset(&domain, 0, sizeof(struct bbox)); + domain.num_dims = ds_conf.ndim; + int i; + for(i = 0; i < domain.num_dims; i++) { + domain.lb.c[i] = 0; + domain.ub.c[i] = ds_conf.dims.c[i] - 1; + } - dsg_l = malloc(sizeof(*dsg_l)); - if (!dsg_l) - goto err_out; + dsg_l = malloc(sizeof(*dsg_l)); + if(!dsg_l) + goto err_out; - MPI_Comm_size(comm, &(dsg_l->size_sp)); + MPI_Comm_size(comm, &(dsg_l->size_sp)); - MPI_Comm_rank(comm, &dsg_l->rank); + MPI_Comm_rank(comm, &dsg_l->rank); - write_conf(server, comm); + write_conf(server, comm); - err = init_sspace(&domain, dsg_l); - if (err < 0) { - goto err_free; - } - dsg_l->ls = ls_alloc(ds_conf.max_versions); - if (!dsg_l->ls) { - fprintf(stderr, "%s(): ERROR ls_alloc() failed\n", __func__); - goto err_free; - } + err = init_sspace(&domain, dsg_l); + if(err < 0) { + goto err_free; + } + dsg_l->ls = ls_alloc(ds_conf.max_versions); + if(!dsg_l->ls) { + fprintf(stderr, "%s(): ERROR ls_alloc() failed\n", __func__); + goto err_free; + } - dsg_l->num_apps = ds_conf.num_apps; + dsg_l->num_apps = ds_conf.num_apps; - INIT_LIST_HEAD(&dsg_l->obj_desc_drain_list); - - server->dsg = dsg_l; - return 0; - err_free: - free(dsg_l); - err_out: - fprintf(stderr, "'%s()': failed with %d.\n", __func__, err); - return err; + INIT_LIST_HEAD(&dsg_l->obj_desc_drain_list); + + server->dsg = dsg_l; + return 0; +err_free: + free(dsg_l); +err_out: + fprintf(stderr, "'%s()': failed with %d.\n", __func__, err); + return err; } static int free_sspace(struct ds_gspace *dsg_l) @@ -405,7 +414,7 @@ static int free_sspace(struct ds_gspace *dsg_l) ssd_free(dsg_l->ssd); struct sspace_list_entry *ssd_entry, *temp; list_for_each_entry_safe(ssd_entry, temp, &dsg_l->sspace_list, - struct sspace_list_entry, entry) + struct sspace_list_entry, entry) { ssd_free(ssd_entry->ssd); list_del(&ssd_entry->entry); @@ -415,30 +424,30 @@ static int free_sspace(struct ds_gspace *dsg_l) return 0; } - -static struct sspace* lookup_sspace(struct ds_gspace *dsg_l, const char* var_name, const struct global_dimension* gd) +static struct sspace *lookup_sspace(struct ds_gspace *dsg_l, + const char *var_name, + const struct global_dimension *gd) { struct global_dimension gdim; memcpy(&gdim, gd, sizeof(struct global_dimension)); // Return the default shared space created based on - // global data domain specified in dataspaces.conf - if (global_dimension_equal(&gdim, &dsg_l->default_gdim )) { + // global data domain specified in dataspaces.conf + if(global_dimension_equal(&gdim, &dsg_l->default_gdim)) { return dsg_l->ssd; } - // Otherwise, search for shared space based on the // global data domain specified by application in put()/get(). struct sspace_list_entry *ssd_entry = NULL; list_for_each_entry(ssd_entry, &dsg_l->sspace_list, - struct sspace_list_entry, entry) + struct sspace_list_entry, entry) { // compare global dimension - if (gdim.ndim != ssd_entry->gdim.ndim) + if(gdim.ndim != ssd_entry->gdim.ndim) continue; - if (global_dimension_equal(&gdim, &ssd_entry->gdim)) + if(global_dimension_equal(&gdim, &ssd_entry->gdim)) return ssd_entry->ssd; } @@ -447,25 +456,25 @@ static struct sspace* lookup_sspace(struct ds_gspace *dsg_l, const char* var_nam struct bbox domain; memset(&domain, 0, sizeof(struct bbox)); domain.num_dims = gdim.ndim; - for (i = 0; i < gdim.ndim; i++) { + for(i = 0; i < gdim.ndim; i++) { domain.lb.c[i] = 0; domain.ub.c[i] = gdim.sizes.c[i] - 1; - } + } - ssd_entry = malloc(sizeof(struct sspace_list_entry)); memcpy(&ssd_entry->gdim, &gdim, sizeof(struct global_dimension)); - ssd_entry->ssd = ssd_alloc(&domain, dsg_l->size_sp, - ds_conf.max_versions, ds_conf.hash_version); - if (!ssd_entry->ssd) { - fprintf(stderr, "%s(): ssd_alloc failed for '%s'\n", __func__, var_name); + ssd_entry->ssd = ssd_alloc(&domain, dsg_l->size_sp, ds_conf.max_versions, + ds_conf.hash_version); + if(!ssd_entry->ssd) { + fprintf(stderr, "%s(): ssd_alloc failed for '%s'\n", __func__, + var_name); return dsg_l->ssd; } err = ssd_init(ssd_entry->ssd, dsg_l->rank); - if (err < 0) { - fprintf(stderr,"%s(): ssd_init failed\n", __func__); + if(err < 0) { + fprintf(stderr, "%s(): ssd_init failed\n", __func__); return dsg_l->ssd; } @@ -473,11 +482,12 @@ static struct sspace* lookup_sspace(struct ds_gspace *dsg_l, const char* var_nam return ssd_entry->ssd; } -static int obj_update_dht(dspaces_provider_t server, struct obj_data *od, obj_update_t type) +static int obj_update_dht(dspaces_provider_t server, struct obj_data *od, + obj_update_t type) { obj_descriptor *odsc = &od->obj_desc; ABT_mutex_lock(server->sspace_mutex); - struct sspace* ssd = lookup_sspace(server->dsg, odsc->name, &od->gdim); + struct sspace *ssd = lookup_sspace(server->dsg, odsc->name, &od->gdim); ABT_mutex_unlock(server->sspace_mutex); struct dht_entry *dht_tab[ssd->dht->num_entries]; @@ -485,13 +495,13 @@ static int obj_update_dht(dspaces_provider_t server, struct obj_data *od, obj_up /* Compute object distribution to nodes in the space. */ num_de = ssd_hash(ssd, &odsc->bb, dht_tab); - if (num_de == 0) { + if(num_de == 0) { fprintf(stderr, "'%s()': this should not happen, num_de == 0 ?!\n", - __func__); + __func__); } /* Update object descriptors on the corresponding nodes. */ - for (i = 0; i < num_de; i++) { - if (dht_tab[i]->rank == server->dsg->rank) { + for(i = 0; i < num_de; i++) { + if(dht_tab[i]->rank == server->dsg->rank) { DEBUG_OUT("Add in local_dht %d\n", server->dsg->rank); ABT_mutex_lock(server->dht_mutex); switch(type) { @@ -502,40 +512,44 @@ static int obj_update_dht(dspaces_provider_t server, struct obj_data *od, obj_up dht_update_owner(ssd->ent_self, odsc); break; default: - fprintf(stderr, "ERROR: (%s): unknown object update type.\n", __func__); + fprintf(stderr, "ERROR: (%s): unknown object update type.\n", + __func__); } ABT_mutex_unlock(server->dht_mutex); DEBUG_OUT("I am self, added in local dht %d\n", server->dsg->rank); continue; } - //now send rpc to the server for dht_update + // now send rpc to the server for dht_update hg_return_t hret; odsc_gdim_t in; - DEBUG_OUT("Server %d sending object %s to dht server %d \n", server->dsg->rank, obj_desc_sprint(odsc), dht_tab[i]->rank); + DEBUG_OUT("Server %d sending object %s to dht server %d \n", + server->dsg->rank, obj_desc_sprint(odsc), dht_tab[i]->rank); in.odsc_gdim.size = sizeof(*odsc); in.odsc_gdim.gdim_size = sizeof(struct global_dimension); - in.odsc_gdim.raw_odsc = (char*)(odsc); - in.odsc_gdim.raw_gdim = (char*)(&od->gdim); + in.odsc_gdim.raw_odsc = (char *)(odsc); + in.odsc_gdim.raw_gdim = (char *)(&od->gdim); in.param = type; hg_addr_t svr_addr; - margo_addr_lookup(server->mid, server->server_address[dht_tab[i]->rank], &svr_addr); + margo_addr_lookup(server->mid, server->server_address[dht_tab[i]->rank], + &svr_addr); hg_handle_t h; margo_create(server->mid, svr_addr, server->obj_update_id, &h); margo_forward(h, &in); - DEBUG_OUT("sent obj server %d to update dht %s in \n", dht_tab[i]->rank, obj_desc_sprint(odsc)); + DEBUG_OUT("sent obj server %d to update dht %s in \n", dht_tab[i]->rank, + obj_desc_sprint(odsc)); margo_addr_free(server->mid, svr_addr); hret = margo_destroy(h); if(hret != HG_SUCCESS) { - fprintf(stderr, "ERROR: (%s): could not destroy handle!\n", __func__); - return(dspaces_ERR_MERCURY); + fprintf(stderr, "ERROR: (%s): could not destroy handle!\n", + __func__); + return (dspaces_ERR_MERCURY); } return dspaces_SUCCESS; - } return 0; @@ -551,37 +565,34 @@ static int get_client_data(obj_descriptor odsc, dspaces_provider_t server) od = obj_data_alloc(&odsc); in.odsc.size = sizeof(obj_descriptor); - in.odsc.raw_odsc = (char*)(&odsc); + in.odsc.raw_odsc = (char *)(&odsc); hg_addr_t owner_addr; size_t owner_addr_size = 128; margo_addr_self(server->mid, &owner_addr); - margo_addr_to_string(server->mid, od->obj_desc.owner, &owner_addr_size, owner_addr); + margo_addr_to_string(server->mid, od->obj_desc.owner, &owner_addr_size, + owner_addr); margo_addr_free(server->mid, owner_addr); + hg_size_t rdma_size = (odsc.size) * bbox_volume(&odsc.bb); - hg_size_t rdma_size = (odsc.size)*bbox_volume(&odsc.bb); - - margo_bulk_create(server->mid, 1, (void**)(&(od->data)), &rdma_size, - HG_BULK_WRITE_ONLY, &in.handle); + margo_bulk_create(server->mid, 1, (void **)(&(od->data)), &rdma_size, + HG_BULK_WRITE_ONLY, &in.handle); hg_addr_t client_addr; margo_addr_lookup(server->mid, odsc.owner, &client_addr); hg_handle_t handle; - margo_create(server->mid, - client_addr, - server->drain_id, - &handle); + margo_create(server->mid, client_addr, server->drain_id, &handle); margo_forward(handle, &in); margo_get_output(handle, &out); - if(out.ret == dspaces_SUCCESS){ + if(out.ret == dspaces_SUCCESS) { ABT_mutex_lock(server->ls_mutex); ls_add_obj(server->dsg->ls, od); ABT_mutex_unlock(server->ls_mutex); } ret = out.ret; - //now update the dht with new owner information + // now update the dht with new owner information DEBUG_OUT("Inside get_client_data\n"); margo_addr_free(server->mid, client_addr); margo_bulk_free(in.handle); @@ -589,11 +600,9 @@ static int get_client_data(obj_descriptor odsc, dspaces_provider_t server) margo_destroy(handle); obj_update_dht(server, od, DS_OBJ_OWNER); return ret; - } - -//thread to move data between layers +// thread to move data between layers static void drain_thread(void *arg) { dspaces_provider_t server = arg; @@ -601,47 +610,48 @@ static void drain_thread(void *arg) while(server->f_kill > 0) { int counter = 0; DEBUG_OUT("Thread WOKEUP\n"); - do{ + do { counter = 0; obj_descriptor odsc; struct obj_desc_list *odscl; - //requires better way to get the obj_descriptor + // requires better way to get the obj_descriptor ABT_mutex_lock(server->odsc_mutex); DEBUG_OUT("Inside odsc mutex\n"); - list_for_each_entry(odscl, &(server->dsg->obj_desc_drain_list), struct obj_desc_list, odsc_entry){ + list_for_each_entry(odscl, &(server->dsg->obj_desc_drain_list), + struct obj_desc_list, odsc_entry) + { memcpy(&odsc, &(odscl->odsc), sizeof(obj_descriptor)); DEBUG_OUT("Found %s in odsc_list\n", obj_desc_sprint(&odsc)); counter = 1; break; - } if(counter == 1) { list_del(&odscl->odsc_entry); ABT_mutex_unlock(server->odsc_mutex); int ret = get_client_data(odsc, server); DEBUG_OUT("Finished draining %s\n", obj_desc_sprint(&odsc)); - if(ret != dspaces_SUCCESS){ + if(ret != dspaces_SUCCESS) { ABT_mutex_lock(server->odsc_mutex); DEBUG_OUT("Drain failed, returning object to queue...\n"); - list_add_tail(&odscl->odsc_entry, &server->dsg->obj_desc_drain_list); + list_add_tail(&odscl->odsc_entry, + &server->dsg->obj_desc_drain_list); ABT_mutex_unlock(server->odsc_mutex); } sleep(1); } else { ABT_mutex_unlock(server->odsc_mutex); } - - }while(counter == 1); - sleep(10); - - ABT_thread_yield(); + } while(counter == 1); + + sleep(10); + + ABT_thread_yield(); } - } - -int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, dspaces_provider_t* sv) +int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, + dspaces_provider_t *sv) { const char *envdebug = getenv("DSPACES_DEBUG"); const char *envnthreads = getenv("DSPACES_NUM_HANDLERS"); @@ -650,11 +660,11 @@ int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, dspaces_provider_t hg_bool_t flag; hg_id_t id; int num_handlers = DSPACES_DEFAULT_NUM_HANDLERS; - int ret; + int ret; server = (dspaces_provider_t)calloc(1, sizeof(*server)); if(server == NULL) - return dspaces_ERR_ALLOCATION; + return dspaces_ERR_ALLOCATION; if(envdebug) { server->f_debug = 1; @@ -671,7 +681,8 @@ int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, dspaces_provider_t MPI_Comm_dup(comm, &server->comm); MPI_Comm_rank(comm, &server->rank); - server->mid = margo_init(listen_addr_str, MARGO_SERVER_MODE, 1, num_handlers); + server->mid = + margo_init(listen_addr_str, MARGO_SERVER_MODE, 1, num_handlers); assert(server->mid); server->listen_addr_str = strdup(listen_addr_str); @@ -681,49 +692,63 @@ int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, dspaces_provider_t ret = ABT_mutex_create(&server->dht_mutex); ret = ABT_mutex_create(&server->sspace_mutex); ret = ABT_mutex_create(&server->kill_mutex); - + margo_registered_name(server->mid, "put_rpc", &id, &flag); if(flag == HG_TRUE) { /* RPCs already registered */ - margo_registered_name(server->mid, "put_rpc", &server->put_id, &flag); - margo_registered_name(server->mid, "put_local_rpc", &server->put_local_id, &flag); - margo_registered_name(server->mid, "get_rpc", &server->get_id, &flag); - margo_registered_name(server->mid, "query_rpc", &server->query_id, &flag); - margo_registered_name(server->mid, "obj_update_rpc", &server->obj_update_id, &flag); - margo_registered_name(server->mid, "odsc_internal_rpc", &server->odsc_internal_id, &flag); - margo_registered_name(server->mid, "ss_rpc", &server->ss_id, &flag); - margo_registered_name(server->mid, "drain_rpc", &server->drain_id, &flag); - margo_registered_name(server->mid, "kill_rpc", &server->kill_id, &flag); - margo_registered_name(server->mid, "sub_rpc", &server->sub_id, &flag); - margo_registered_name(server->mid, "notify_rpc", &server->notify_id, &flag); + margo_registered_name(server->mid, "put_rpc", &server->put_id, &flag); + margo_registered_name(server->mid, "put_local_rpc", + &server->put_local_id, &flag); + margo_registered_name(server->mid, "get_rpc", &server->get_id, &flag); + margo_registered_name(server->mid, "query_rpc", &server->query_id, + &flag); + margo_registered_name(server->mid, "obj_update_rpc", + &server->obj_update_id, &flag); + margo_registered_name(server->mid, "odsc_internal_rpc", + &server->odsc_internal_id, &flag); + margo_registered_name(server->mid, "ss_rpc", &server->ss_id, &flag); + margo_registered_name(server->mid, "drain_rpc", &server->drain_id, + &flag); + margo_registered_name(server->mid, "kill_rpc", &server->kill_id, &flag); + margo_registered_name(server->mid, "sub_rpc", &server->sub_id, &flag); + margo_registered_name(server->mid, "notify_rpc", &server->notify_id, + &flag); } else { - server->put_id = - MARGO_REGISTER(server->mid, "put_rpc", bulk_gdim_t, bulk_out_t, put_rpc); - margo_register_data(server->mid, server->put_id, (void*)server, NULL); + server->put_id = MARGO_REGISTER(server->mid, "put_rpc", bulk_gdim_t, + bulk_out_t, put_rpc); + margo_register_data(server->mid, server->put_id, (void *)server, NULL); server->put_local_id = - MARGO_REGISTER(server->mid, "put_local_rpc", odsc_gdim_t, bulk_out_t, put_local_rpc); - margo_register_data(server->mid, server->put_local_id, (void*)server, NULL); - server->get_id = - MARGO_REGISTER(server->mid, "get_rpc", bulk_in_t, bulk_out_t, get_rpc); - margo_register_data(server->mid, server->get_id, (void*)server, NULL); - server->query_id = - MARGO_REGISTER(server->mid, "query_rpc", odsc_gdim_t, odsc_list_t, query_rpc); - margo_register_data(server->mid, server->query_id, (void*)server, NULL); - server->obj_update_id = - MARGO_REGISTER(server->mid, "obj_update_rpc", odsc_gdim_t, void, obj_update_rpc); - margo_register_data(server->mid, server->obj_update_id, (void*)server, NULL); - margo_registered_disable_response(server->mid, server->obj_update_id, HG_TRUE); + MARGO_REGISTER(server->mid, "put_local_rpc", odsc_gdim_t, + bulk_out_t, put_local_rpc); + margo_register_data(server->mid, server->put_local_id, (void *)server, + NULL); + server->get_id = MARGO_REGISTER(server->mid, "get_rpc", bulk_in_t, + bulk_out_t, get_rpc); + margo_register_data(server->mid, server->get_id, (void *)server, NULL); + server->query_id = MARGO_REGISTER(server->mid, "query_rpc", odsc_gdim_t, + odsc_list_t, query_rpc); + margo_register_data(server->mid, server->query_id, (void *)server, + NULL); + server->obj_update_id = MARGO_REGISTER( + server->mid, "obj_update_rpc", odsc_gdim_t, void, obj_update_rpc); + margo_register_data(server->mid, server->obj_update_id, (void *)server, + NULL); + margo_registered_disable_response(server->mid, server->obj_update_id, + HG_TRUE); server->odsc_internal_id = - MARGO_REGISTER(server->mid, "odsc_internal_rpc", odsc_gdim_t, odsc_list_t, odsc_internal_rpc); - margo_register_data(server->mid, server->odsc_internal_id, (void*)server, NULL); + MARGO_REGISTER(server->mid, "odsc_internal_rpc", odsc_gdim_t, + odsc_list_t, odsc_internal_rpc); + margo_register_data(server->mid, server->odsc_internal_id, + (void *)server, NULL); server->ss_id = MARGO_REGISTER(server->mid, "ss_rpc", void, ss_information, ss_rpc); - margo_register_data(server->mid, server->ss_id, (void*)server, NULL); - server->drain_id = - MARGO_REGISTER(server->mid, "drain_rpc", bulk_in_t, bulk_out_t, NULL); + margo_register_data(server->mid, server->ss_id, (void *)server, NULL); + server->drain_id = MARGO_REGISTER(server->mid, "drain_rpc", bulk_in_t, + bulk_out_t, NULL); server->kill_id = MARGO_REGISTER(server->mid, "kill_rpc", int32_t, void, kill_rpc); - margo_registered_disable_response(server->mid, server->kill_id, HG_TRUE); + margo_registered_disable_response(server->mid, server->kill_id, + HG_TRUE); margo_register_data(server->mid, server->kill_id, (void *)server, NULL); server->sub_id = MARGO_REGISTER(server->mid, "sub_rpc", odsc_gdim_t, void, sub_rpc); @@ -731,19 +756,22 @@ int dspaces_server_init(char *listen_addr_str, MPI_Comm comm, dspaces_provider_t margo_registered_disable_response(server->mid, server->sub_id, HG_TRUE); server->notify_id = MARGO_REGISTER(server->mid, "notify_rpc", odsc_list_t, void, NULL); - margo_registered_disable_response(server->mid, server->notify_id, HG_TRUE); + margo_registered_disable_response(server->mid, server->notify_id, + HG_TRUE); } int size_sp = 1; int err = dsg_alloc(server, "dataspaces.conf", comm); - assert(err == 0); + assert(err == 0); server->f_kill = server->dsg->num_apps; if(server->f_drain) { - //thread to drain the data + // thread to drain the data ABT_xstream_create(ABT_SCHED_NULL, &server->drain_xstream); - ABT_xstream_get_main_pools(server->drain_xstream, 1, &server->drain_pool); - ABT_thread_create(server->drain_pool, drain_thread, server, ABT_THREAD_ATTR_NULL, &server->drain_t); + ABT_xstream_get_main_pools(server->drain_xstream, 1, + &server->drain_pool); + ABT_thread_create(server->drain_pool, drain_thread, server, + ABT_THREAD_ATTR_NULL, &server->drain_t); } *sv = server; @@ -765,7 +793,8 @@ static void kill_client(dspaces_provider_t server, char *client_addr) } /* - * Clients with local data need to know when it's safe to finalize. Send kill rpc to any clients in the drain list. + * Clients with local data need to know when it's safe to finalize. Send kill + * rpc to any clients in the drain list. */ static void kill_local_clients(dspaces_provider_t server) { @@ -779,9 +808,13 @@ static void kill_local_clients(dspaces_provider_t server) DEBUG_OUT("Killing clients with local storage.\n"); ABT_mutex_lock(server->odsc_mutex); - list_for_each_entry(odscl, &(server->dsg->obj_desc_drain_list), struct obj_desc_list, odsc_entry) { + list_for_each_entry(odscl, &(server->dsg->obj_desc_drain_list), + struct obj_desc_list, odsc_entry) + { found = 0; - list_for_each_entry(client_addr, &client_list, struct addr_list_entry, entry) { + list_for_each_entry(client_addr, &client_list, struct addr_list_entry, + entry) + { if(strcmp(client_addr->addr, odscl->odsc.owner) == 0) { found = 1; break; @@ -796,7 +829,9 @@ static void kill_local_clients(dspaces_provider_t server) } ABT_mutex_unlock(server->odsc_mutex); - list_for_each_entry_safe(client_addr, temp, &client_list, struct addr_list_entry, entry) { + list_for_each_entry_safe(client_addr, temp, &client_list, + struct addr_list_entry, entry) + { DEBUG_OUT("Sending kill signal to %s.\n", client_addr->addr); kill_client(server, client_addr->addr); list_del(&client_addr->entry); @@ -811,7 +846,8 @@ static int server_destroy(dspaces_provider_t server) MPI_Barrier(server->comm); if(server->rank == 0) { - fprintf(stderr, "Finishing up, waiting for asynchronous jobs to finish...\n"); + fprintf(stderr, + "Finishing up, waiting for asynchronous jobs to finish...\n"); } if(server->f_drain) { @@ -847,11 +883,13 @@ static void put_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); + if(server->f_kill == 0) { - fprintf(stderr, "WARNING: put rpc received when server is finalizing. This will likely cause problems...\n"); + fprintf(stderr, "WARNING: put rpc received when server is finalizing. " + "This will likely cause problems...\n"); } hret = margo_get_input(handle, &in); @@ -859,12 +897,13 @@ static void put_rpc(hg_handle_t handle) obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc.raw_odsc, sizeof(in_odsc)); - //set the owner to be this server address + // set the owner to be this server address hg_addr_t owner_addr; size_t owner_addr_size = 128; margo_addr_self(server->mid, &owner_addr); - margo_addr_to_string(server->mid, in_odsc.owner, &owner_addr_size, owner_addr); + margo_addr_to_string(server->mid, in_odsc.owner, &owner_addr_size, + owner_addr); margo_addr_free(server->mid, owner_addr); struct obj_data *od; @@ -874,13 +913,13 @@ static void put_rpc(hg_handle_t handle) if(!od) fprintf(stderr, "ERROR: (%s): object allocation failed!\n", __func__); - //do write lock + // do write lock - hg_size_t size = (in_odsc.size)*bbox_volume(&(in_odsc.bb)); + hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); - void *buffer = (void*) od->data; - hret = margo_bulk_create(mid, 1, (void**)&(od->data), &size, - HG_BULK_WRITE_ONLY, &bulk_handle); + void *buffer = (void *)od->data; + hret = margo_bulk_create(mid, 1, (void **)&(od->data), &size, + HG_BULK_WRITE_ONLY, &bulk_handle); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: (%s): margo_bulk_create failed!\n", __func__); @@ -889,10 +928,10 @@ static void put_rpc(hg_handle_t handle) margo_free_input(handle, &in); margo_destroy(handle); return; - } - + } + hret = margo_bulk_transfer(mid, HG_BULK_PULL, info->addr, in.handle, 0, - bulk_handle, 0, size); + bulk_handle, 0, size); if(hret != HG_SUCCESS) { fprintf(stderr, "ERROR: (%s): margo_bulk_transfer failed!\n", __func__); out.ret = dspaces_ERR_MERCURY; @@ -909,7 +948,7 @@ static void put_rpc(hg_handle_t handle) DEBUG_OUT("Received obj %s\n", obj_desc_sprint(&od->obj_desc)); - //now update the dht + // now update the dht out.ret = dspaces_SUCCESS; margo_bulk_free(bulk_handle); margo_respond(handle, &out); @@ -918,11 +957,9 @@ static void put_rpc(hg_handle_t handle) obj_update_dht(server, od, DS_OBJ_NEW); DEBUG_OUT("Finished obj_put_update from put_rpc\n"); - } DEFINE_MARGO_RPC_HANDLER(put_rpc) - static void put_local_rpc(hg_handle_t handle) { hg_return_t hret; @@ -931,13 +968,17 @@ static void put_local_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); DEBUG_OUT("In the local put rpc\n"); if(server->f_kill == 0) { - fprintf(stderr, "WARNING: (%s): got put rpc with local storage, but server is shutting down. This will likely cause problems...\n", __func__); + fprintf(stderr, + "WARNING: (%s): got put rpc with local storage, but server is " + "shutting down. This will likely cause problems...\n", + __func__); } hret = margo_get_input(handle, &in); @@ -951,17 +992,17 @@ static void put_local_rpc(hg_handle_t handle) memcpy(&od->gdim, in.odsc_gdim.raw_gdim, sizeof(struct global_dimension)); if(!od) - fprintf(stderr, "ERROR: (%s): failed to allocate object data!\n", __func__); + fprintf(stderr, "ERROR: (%s): failed to allocate object data!\n", + __func__); + DEBUG_OUT("Received obj %s in put_local_rpc\n", + obj_desc_sprint(&od->obj_desc)); - DEBUG_OUT("Received obj %s in put_local_rpc\n", obj_desc_sprint(&od->obj_desc)); - - - //now update the dht + // now update the dht obj_update_dht(server, od, DS_OBJ_NEW); DEBUG_OUT("Finished obj_put_local_update in local_put\n"); - //add to the local list for marking as to be drained data + // add to the local list for marking as to be drained data struct obj_desc_list *odscl; odscl = malloc(sizeof(*odscl)); memcpy(&odscl->odsc, &od->obj_desc, sizeof(obj_descriptor)); @@ -971,17 +1012,18 @@ static void put_local_rpc(hg_handle_t handle) list_add_tail(&odscl->odsc_entry, &server->dsg->obj_desc_drain_list); ABT_mutex_unlock(server->odsc_mutex); - //TODO: wake up thread to initiate draining + // TODO: wake up thread to initiate draining out.ret = dspaces_SUCCESS; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); - + free(od); } DEFINE_MARGO_RPC_HANDLER(put_local_rpc) -static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, int timeout, obj_descriptor **results) +static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, + int timeout, obj_descriptor **results) { struct sspace *ssd; struct dht_entry **de_tab; @@ -1024,22 +1066,28 @@ static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, int ti self_id_num = i; continue; } - //remote servers - margo_addr_lookup(server->mid, server->server_address[de_tab[i]->rank], &server_addr); - margo_create(server->mid, server_addr, server->odsc_internal_id, &hndls[i]); + // remote servers + margo_addr_lookup(server->mid, server->server_address[de_tab[i]->rank], + &server_addr); + margo_create(server->mid, server_addr, server->odsc_internal_id, + &hndls[i]); margo_iforward(hndls[i], query, &serv_reqs[i]); margo_addr_free(server->mid, server_addr); } if(self_id_num > -1) { podsc = malloc(sizeof(*podsc) * ssd->ent_self->odsc_num); - odsc_nums[self_id_num] = dht_find_entry_all(ssd->ent_self, q_odsc, &podsc, timeout); - DEBUG_OUT("%d odscs found in %d\n", odsc_nums[self_id_num], server->dsg->rank); + odsc_nums[self_id_num] = + dht_find_entry_all(ssd->ent_self, q_odsc, &podsc, timeout); + DEBUG_OUT("%d odscs found in %d\n", odsc_nums[self_id_num], + server->dsg->rank); total_odscs += odsc_nums[self_id_num]; if(odsc_nums[self_id_num]) { - odsc_tabs[self_id_num] = malloc(sizeof(**odsc_tabs) * odsc_nums[self_id_num]); + odsc_tabs[self_id_num] = + malloc(sizeof(**odsc_tabs) * odsc_nums[self_id_num]); for(i = 0; i < odsc_nums[self_id_num]; i++) { - obj_descriptor *odsc = &odsc_tabs[self_id_num][i]; //readability + obj_descriptor *odsc = + &odsc_tabs[self_id_num][i]; // readability *odsc = *podsc[i]; odsc->st = q_odsc->st; bbox_intersect(&q_odsc->bb, &odsc->bb, &odsc->bb); @@ -1050,7 +1098,6 @@ static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, int ti free(podsc); } - for(i = 0; i < peer_num; i++) { if(i == self_id_num) { continue; @@ -1063,11 +1110,13 @@ static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, int ti DEBUG_OUT("received %d odscs from peer %d\n", odsc_nums[i], i); total_odscs += odsc_nums[i]; odsc_tabs[i] = malloc(sizeof(**odsc_tabs) * odsc_nums[i]); - memcpy(odsc_tabs[i], dht_resp.odsc_list.raw_odsc, dht_resp.odsc_list.size); + memcpy(odsc_tabs[i], dht_resp.odsc_list.raw_odsc, + dht_resp.odsc_list.size); for(j = 0; j < odsc_nums[i]; j++) { - //readability - obj_descriptor *odsc = (obj_descriptor *)dht_resp.odsc_list.raw_odsc; + // readability + obj_descriptor *odsc = + (obj_descriptor *)dht_resp.odsc_list.raw_odsc; DEBUG_OUT("remote buffer: %s\n", obj_desc_sprint(&odsc[j])); } } @@ -1096,7 +1145,7 @@ static int get_query_odscs(dspaces_provider_t server, odsc_gdim_t *query, int ti free(odsc_tabs); free(odsc_nums); - return(sizeof(obj_descriptor) * total_odscs); + return (sizeof(obj_descriptor) * total_odscs); } static void query_rpc(hg_handle_t handle) @@ -1124,7 +1173,8 @@ static void query_rpc(hg_handle_t handle) memcpy(&in_odsc, in.odsc_gdim.raw_odsc, sizeof(in_odsc)); memcpy(&in_gdim, in.odsc_gdim.raw_gdim, sizeof(struct global_dimension)); timeout = in.param; - DEBUG_OUT("Received query for %s with timeout %d", obj_desc_sprint(&in_odsc), timeout); + DEBUG_OUT("Received query for %s with timeout %d", + obj_desc_sprint(&in_odsc), timeout); out.odsc_list.size = get_query_odscs(server, &in, timeout, &results); @@ -1146,43 +1196,44 @@ static void get_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + assert(hret == HG_SUCCESS); obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc.raw_odsc, sizeof(in_odsc)); DEBUG_OUT("received get request\n"); - + struct obj_data *od, *from_obj; from_obj = ls_find(server->dsg->ls, &in_odsc); od = obj_data_alloc(&in_odsc); ssd_copy(od, from_obj); - - hg_size_t size = (in_odsc.size)*bbox_volume(&(in_odsc.bb)); - void *buffer = (void*) od->data; - hret = margo_bulk_create(mid, 1, (void**)&buffer, &size, - HG_BULK_READ_ONLY, &bulk_handle); + + hg_size_t size = (in_odsc.size) * bbox_volume(&(in_odsc.bb)); + void *buffer = (void *)od->data; + hret = margo_bulk_create(mid, 1, (void **)&buffer, &size, HG_BULK_READ_ONLY, + &bulk_handle); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_create() failure\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_create() failure\n", __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); return; - } + } hret = margo_bulk_transfer(mid, HG_BULK_PUSH, info->addr, in.handle, 0, - bulk_handle, 0, size); + bulk_handle, 0, size); if(hret != HG_SUCCESS) { - fprintf(stderr,"ERROR: (%s): margo_bulk_transfer() failure\n", __func__); + fprintf(stderr, "ERROR: (%s): margo_bulk_transfer() failure\n", + __func__); out.ret = dspaces_ERR_MERCURY; margo_respond(handle, &out); margo_free_input(handle, &in); @@ -1199,7 +1250,6 @@ static void get_rpc(hg_handle_t handle) } DEFINE_MARGO_RPC_HANDLER(get_rpc) - static void odsc_internal_rpc(hg_handle_t handle) { hg_return_t hret; @@ -1209,8 +1259,9 @@ static void odsc_internal_rpc(hg_handle_t handle) obj_descriptor **podsc; margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); hret = margo_get_input(handle, &in); assert(hret == HG_SUCCESS); @@ -1222,27 +1273,28 @@ static void odsc_internal_rpc(hg_handle_t handle) struct global_dimension od_gdim; memcpy(&od_gdim, in.odsc_gdim.raw_gdim, sizeof(struct global_dimension)); - DEBUG_OUT("Received query for %s with timeout %d\n", obj_desc_sprint(&in_odsc), timeout); - + DEBUG_OUT("Received query for %s with timeout %d\n", + obj_desc_sprint(&in_odsc), timeout); + obj_descriptor odsc, *odsc_tab; ABT_mutex_lock(server->sspace_mutex); - struct sspace* ssd = lookup_sspace(server->dsg, in_odsc.name, &od_gdim); + struct sspace *ssd = lookup_sspace(server->dsg, in_odsc.name, &od_gdim); ABT_mutex_unlock(server->sspace_mutex); podsc = malloc(sizeof(*podsc) * ssd->ent_self->odsc_num); int num_odsc; num_odsc = dht_find_entry_all(ssd->ent_self, &in_odsc, &podsc, timeout); DEBUG_OUT("found %d DHT entries.\n", num_odsc); - if (!num_odsc) { - //need to figure out how to send that number of odscs is null + if(!num_odsc) { + // need to figure out how to send that number of odscs is null out.odsc_list.size = 0; out.odsc_list.raw_odsc = NULL; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); - - }else{ + + } else { odsc_tab = malloc(sizeof(*odsc_tab) * num_odsc); - for (int j = 0; j < num_odsc; j++) { + for(int j = 0; j < num_odsc; j++) { obj_descriptor odsc; odsc = *podsc[j]; DEBUG_OUT("including %s\n", obj_desc_sprint(&odsc)); @@ -1252,14 +1304,12 @@ static void odsc_internal_rpc(hg_handle_t handle) odsc_tab[j] = odsc; } out.odsc_list.size = num_odsc * sizeof(obj_descriptor); - out.odsc_list.raw_odsc = (char*)odsc_tab; + out.odsc_list.raw_odsc = (char *)odsc_tab; margo_respond(handle, &out); margo_free_input(handle, &in); margo_destroy(handle); - } - for (int i = 0; i < num_odsc; ++i) - { + for(int i = 0; i < num_odsc; ++i) { DEBUG_OUT("send odsc: %s\n", obj_desc_sprint(&odsc_tab[i])); } @@ -1280,13 +1330,14 @@ static void obj_update_rpc(hg_handle_t handle) margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); DEBUG_OUT("Received rpc to update obj_dht\n"); hret = margo_get_input(handle, &in); - assert(hret == HG_SUCCESS); + assert(hret == HG_SUCCESS); obj_descriptor in_odsc; memcpy(&in_odsc, in.odsc_gdim.raw_odsc, sizeof(in_odsc)); @@ -1296,13 +1347,13 @@ static void obj_update_rpc(hg_handle_t handle) DEBUG_OUT("received update_rpc %s\n", obj_desc_sprint(&in_odsc)); ABT_mutex_lock(server->sspace_mutex); - struct sspace* ssd = lookup_sspace(server->dsg, in_odsc.name, &gdim); + struct sspace *ssd = lookup_sspace(server->dsg, in_odsc.name, &gdim); ABT_mutex_unlock(server->sspace_mutex); struct dht_entry *de = ssd->ent_self; ABT_mutex_lock(server->dht_mutex); switch(type) { - case DS_OBJ_NEW: + case DS_OBJ_NEW: err = dht_add_entry(de, &in_odsc); break; case DS_OBJ_OWNER: @@ -1312,40 +1363,42 @@ static void obj_update_rpc(hg_handle_t handle) fprintf(stderr, "ERROR: (%s): unknown object update type.\n", __func__); } ABT_mutex_unlock(server->dht_mutex); - DEBUG_OUT("Updated dht %s in server %d \n", obj_desc_sprint(&in_odsc), server->dsg->rank); - if (err < 0) - fprintf(stderr, "ERROR (%s): obj_update_rpc Failed with %d\n", __func__, err); + DEBUG_OUT("Updated dht %s in server %d \n", obj_desc_sprint(&in_odsc), + server->dsg->rank); + if(err < 0) + fprintf(stderr, "ERROR (%s): obj_update_rpc Failed with %d\n", __func__, + err); margo_free_input(handle, &in); margo_destroy(handle); } DEFINE_MARGO_RPC_HANDLER(obj_update_rpc) - static void ss_rpc(hg_handle_t handle) { ss_information out; margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); ss_info_hdr ss_data; ss_data.num_dims = ds_conf.ndim; ss_data.num_space_srv = server->dsg->size_sp; - ss_data.max_versions = ds_conf.max_versions; + ss_data.max_versions = ds_conf.max_versions; ss_data.hash_version = ds_conf.hash_version; ss_data.default_gdim.ndim = ds_conf.ndim; - for(int i = 0; i < ds_conf.ndim; i++){ + for(int i = 0; i < ds_conf.ndim; i++) { ss_data.ss_domain.lb.c[i] = 0; - ss_data.ss_domain.ub.c[i] = ds_conf.dims.c[i]-1; + ss_data.ss_domain.ub.c[i] = ds_conf.dims.c[i] - 1; ss_data.default_gdim.sizes.c[i] = ds_conf.dims.c[i]; } out.ss_buf.size = sizeof(ss_info_hdr); - out.ss_buf.raw_odsc = (char*)(&ss_data); + out.ss_buf.raw_odsc = (char *)(&ss_data); margo_respond(handle, &out); margo_destroy(handle); } @@ -1353,11 +1406,12 @@ DEFINE_MARGO_RPC_HANDLER(ss_rpc) static void send_kill_rpc(dspaces_provider_t server, int target, int *rank) { - //TODO: error handling/reporting + // TODO: error handling/reporting hg_addr_t server_addr; hg_handle_t h; - - margo_addr_lookup(server->mid, server->server_address[target], &server_addr); + + margo_addr_lookup(server->mid, server->server_address[target], + &server_addr); margo_create(server->mid, server_addr, server->kill_id, &h); margo_forward(h, rank); margo_addr_free(server->mid, server_addr); @@ -1367,8 +1421,9 @@ static void send_kill_rpc(dspaces_provider_t server, int target, int *rank) static void kill_rpc(hg_handle_t handle) { margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); int32_t src, rank, parent, child1, child2; int do_kill = 0; hg_return_t hret; @@ -1382,9 +1437,10 @@ static void kill_rpc(hg_handle_t handle) child2 = child1 + 1; ABT_mutex_lock(server->kill_mutex); - DEBUG_OUT("Kill tokens remaining: %d\n", server->f_kill?(server->f_kill-1):0); + DEBUG_OUT("Kill tokens remaining: %d\n", + server->f_kill ? (server->f_kill - 1) : 0); if(server->f_kill == 0) { - //already shutting down + // already shutting down ABT_mutex_unlock(server->kill_mutex); margo_free_input(handle, &src); margo_destroy(handle); @@ -1394,7 +1450,7 @@ static void kill_rpc(hg_handle_t handle) DEBUG_OUT("Kill count is zero. Initiating shutdown.\n"); do_kill = 1; } - + ABT_mutex_unlock(server->kill_mutex); if((src == -1 || src > rank) && rank > 0) { @@ -1418,9 +1474,10 @@ DEFINE_MARGO_RPC_HANDLER(kill_rpc) static void sub_rpc(hg_handle_t handle) { margo_instance_id mid = margo_hg_handle_get_instance(handle); - const struct hg_info* info = margo_get_info(handle); - dspaces_provider_t server = (dspaces_provider_t)margo_registered_data(mid, info->id); - odsc_list_t notice; + const struct hg_info *info = margo_get_info(handle); + dspaces_provider_t server = + (dspaces_provider_t)margo_registered_data(mid, info->id); + odsc_list_t notice; odsc_gdim_t in; int32_t sub_id; hg_return_t hret; @@ -1436,9 +1493,10 @@ static void sub_rpc(hg_handle_t handle) memcpy(&in_gdim, in.odsc_gdim.raw_gdim, sizeof(struct global_dimension)); sub_id = in.param; - DEBUG_OUT("received subscription for %s with id %d from %s\n", obj_desc_sprint(&in_odsc), sub_id, in_odsc.owner); + DEBUG_OUT("received subscription for %s with id %d from %s\n", + obj_desc_sprint(&in_odsc), sub_id, in_odsc.owner); - in.param = -1; // this will be interpreted as timeout by any interal queries + in.param = -1; // this will be interpreted as timeout by any interal queries notice.odsc_list.size = get_query_odscs(server, &in, -1, &results); notice.odsc_list.raw_odsc = (char *)results; notice.param = sub_id; diff --git a/src/ss_data.c b/src/ss_data.c index aa24aa3d..44d71fee 100644 --- a/src/ss_data.c +++ b/src/ss_data.c @@ -1,77 +1,80 @@ /* - * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers University - * All rights reserved. + * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers + * University All rights reserved. * - * Redistribution and use in source and binary forms, with or without modification, are permitted provided - * that the following conditions are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * - Redistributions of source code must retain the above copyright notice, this list of conditions and - * the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and - * the following disclaimer in the documentation and/or other materials provided with the distribution. - * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers University, nor the names of its - * contributors may be used to endorse or promote products derived from this software without specific prior - * written permission. + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers + * University, nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written + * permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* -* Ciprian Docan (2009) TASSL Rutgers University -* docan@cac.rutgers.edu -* -* Pradeep Subedi (2020) Rutgers University -* pradeep.subedi@rutgers.edu -*/ + * Ciprian Docan (2009) TASSL Rutgers University + * docan@cac.rutgers.edu + * + * Pradeep Subedi (2020) Rutgers University + * pradeep.subedi@rutgers.edu + */ -#include -#include #include "ss_data.h" #include "queue.h" +#include +#include -#include +#include /* A view in the matrix allows to extract any subset of values from a matrix. */ - struct matrix_view { - uint64_t lb[BBOX_MAX_NDIM]; - uint64_t ub[BBOX_MAX_NDIM]; + uint64_t lb[BBOX_MAX_NDIM]; + uint64_t ub[BBOX_MAX_NDIM]; }; /* Generic matrix representation. */ struct matrix { - uint64_t dist[BBOX_MAX_NDIM]; - int num_dims; - size_t size_elem; - enum storage_type mat_storage; - struct matrix_view mat_view; - void *pdata; + uint64_t dist[BBOX_MAX_NDIM]; + int num_dims; + size_t size_elem; + enum storage_type mat_storage; + struct matrix_view mat_view; + void *pdata; }; - /* Cache structure to "map" a bounding box to corresponding nodes in the space. */ struct sfc_hash_cache { - struct list_head sh_entry; + struct list_head sh_entry; - struct bbox sh_bb; + struct bbox sh_bb; - struct dht_entry **sh_de_tab; - int sh_nodes; + struct dht_entry **sh_de_tab; + int sh_nodes; }; static LIST_HEAD(sfc_hash_list); @@ -79,104 +82,108 @@ static int is_sfc_hash_list_free = 0; static uint64_t next_pow_2_v2(uint64_t n) { - uint64_t i; + uint64_t i; - i = 1; - while (i < n) { - i = i << 1; - } + i = 1; + while(i < n) { + i = i << 1; + } - return i; + return i; } static int compute_bits_v2(uint64_t n) { - int nr_bits = 0; + int nr_bits = 0; - while (n>1) { - n = n >> 1; - nr_bits++; - } + while(n > 1) { + n = n >> 1; + nr_bits++; + } - return nr_bits; + return nr_bits; } static int compute_bits(uint64_t n) { - int nr_bits = 0; + int nr_bits = 0; - while (n) { - n = n >> 1; - nr_bits++; - } + while(n) { + n = n >> 1; + nr_bits++; + } - return nr_bits; + return nr_bits; } int sh_add(const struct bbox *bb, struct dht_entry *de_tab[], int n) { - struct sfc_hash_cache *shc; - int i, err = -ENOMEM; + struct sfc_hash_cache *shc; + int i, err = -ENOMEM; - shc = malloc(sizeof(*shc) + sizeof(de_tab[0]) * n); - if (!shc) - goto err_out; + shc = malloc(sizeof(*shc) + sizeof(de_tab[0]) * n); + if(!shc) + goto err_out; - shc->sh_bb = *bb; - shc->sh_nodes = n; + shc->sh_bb = *bb; + shc->sh_nodes = n; - shc->sh_de_tab = (struct dht_entry **) (shc+1); - for (i = 0; i < n; i++) - shc->sh_de_tab[i] = de_tab[i]; + shc->sh_de_tab = (struct dht_entry **)(shc + 1); + for(i = 0; i < n; i++) + shc->sh_de_tab[i] = de_tab[i]; - list_add_tail(&shc->sh_entry, &sfc_hash_list); + list_add_tail(&shc->sh_entry, &sfc_hash_list); - return 0; - err_out: - fprintf(stderr, "'%s()': failed with %d.\n", __func__, err); - return err; + return 0; +err_out: + fprintf(stderr, "'%s()': failed with %d.\n", __func__, err); + return err; } int sh_find(const struct bbox *bb, struct dht_entry *de_tab[]) { - struct sfc_hash_cache *shc; - int i; + struct sfc_hash_cache *shc; + int i; - list_for_each_entry(shc, &sfc_hash_list, struct sfc_hash_cache, sh_entry) { - if (bbox_equals(bb, &shc->sh_bb)) { - for (i = 0; i < shc->sh_nodes; i++) - de_tab[i] = shc->sh_de_tab[i]; - return shc->sh_nodes; - } + list_for_each_entry(shc, &sfc_hash_list, struct sfc_hash_cache, sh_entry) + { + if(bbox_equals(bb, &shc->sh_bb)) { + for(i = 0; i < shc->sh_nodes; i++) + de_tab[i] = shc->sh_de_tab[i]; + return shc->sh_nodes; } + } - return -1; + return -1; } void sh_free(void) { - if (is_sfc_hash_list_free) return; + if(is_sfc_hash_list_free) + return; - struct sfc_hash_cache *l, *t; - int n = 0; + struct sfc_hash_cache *l, *t; + int n = 0; - list_for_each_entry_safe(l, t, &sfc_hash_list, struct sfc_hash_cache, sh_entry) { - free(l); - n++; - } + list_for_each_entry_safe(l, t, &sfc_hash_list, struct sfc_hash_cache, + sh_entry) + { + free(l); + n++; + } #ifdef DEBUG - fprintf(stderr, "'%s()': SFC cached %d object descriptors, size = %zu.\n", - __func__, n, sizeof(*l) *n); + fprintf(stderr, "'%s()': SFC cached %d object descriptors, size = %zu.\n", + __func__, n, sizeof(*l) * n); #endif - is_sfc_hash_list_free = 1; + is_sfc_hash_list_free = 1; } static void get_bbox_max_dim(const struct bbox *bb, uint64_t *out_max_dim, - int *out_dim) + int *out_dim) { int i; uint64_t max_dim = 0; - for (i = 0; i < bb->num_dims; i++) { - if (max_dim < bb->ub.c[i]) { + for(i = 0; i < bb->num_dims; i++) { + if(max_dim < bb->ub.c[i]) { max_dim = bb->ub.c[i]; *out_dim = i; } @@ -184,29 +191,28 @@ static void get_bbox_max_dim(const struct bbox *bb, uint64_t *out_max_dim, *out_max_dim = max_dim; } -static void get_bbox_max_dim_size(const struct bbox *bb, uint64_t *out_max_dim_size, - int *out_dim) +static void get_bbox_max_dim_size(const struct bbox *bb, + uint64_t *out_max_dim_size, int *out_dim) { int i; uint64_t max_dim_size = 0; - for (i = 0; i < bb->num_dims; i++) { - uint64_t size = bb->ub.c[i]-bb->lb.c[i]+1; - if (max_dim_size < size) { + for(i = 0; i < bb->num_dims; i++) { + uint64_t size = bb->ub.c[i] - bb->lb.c[i] + 1; + if(max_dim_size < size) { max_dim_size = size; *out_dim = i; } } - *out_max_dim_size = max_dim_size; - + *out_max_dim_size = max_dim_size; } -static struct dht_entry * dht_entry_alloc(struct sspace *ssd, int size_hash) +static struct dht_entry *dht_entry_alloc(struct sspace *ssd, int size_hash) { struct dht_entry *de; int i; - de = malloc(sizeof(*de) + sizeof(struct obj_desc_list)*(size_hash-1)); - if (!de) { + de = malloc(sizeof(*de) + sizeof(struct obj_desc_list) * (size_hash - 1)); + if(!de) { errno = ENOMEM; return de; } @@ -218,7 +224,7 @@ static struct dht_entry * dht_entry_alloc(struct sspace *ssd, int size_hash) de->hash_mutex = malloc(sizeof(*(de->hash_mutex)) * size_hash); de->dht_subs = malloc(sizeof(*(de->dht_subs)) * size_hash); - for (i = 0; i < size_hash; i++) { + for(i = 0; i < size_hash; i++) { INIT_LIST_HEAD(&de->odsc_hash[i]); ABT_cond_create(&de->hash_cond[i]); ABT_mutex_create(&de->hash_mutex[i]); @@ -236,24 +242,24 @@ static void dht_entry_free(struct dht_entry *de) struct obj_desc_list *l, *t; int i; - //TODO: free the *intv and other resources. + // TODO: free the *intv and other resources. free(de->i_tab); - for (i = 0; i < de->odsc_size; i++) { - list_for_each_entry_safe(l, t, &de->odsc_hash[i], struct obj_desc_list, odsc_entry) - free(l); + for(i = 0; i < de->odsc_size; i++) { + list_for_each_entry_safe(l, t, &de->odsc_hash[i], struct obj_desc_list, + odsc_entry) free(l); } free(de); } -static struct dht * -dht_alloc(struct sspace *ssd, const struct bbox *bb_domain, int num_nodes, int size_hash) +static struct dht *dht_alloc(struct sspace *ssd, const struct bbox *bb_domain, + int num_nodes, int size_hash) { struct dht *dht; int i; - dht = malloc(sizeof(*dht) + sizeof(struct dht_entry)*(num_nodes-1)); - if (!dht) { + dht = malloc(sizeof(*dht) + sizeof(struct dht_entry) * (num_nodes - 1)); + if(!dht) { errno = ENOMEM; return dht; } @@ -262,15 +268,15 @@ dht_alloc(struct sspace *ssd, const struct bbox *bb_domain, int num_nodes, int s dht->bb_glb_domain = *bb_domain; dht->num_entries = num_nodes; - for (i = 0; i < num_nodes; i++) { + for(i = 0; i < num_nodes; i++) { dht->ent_tab[i] = dht_entry_alloc(ssd, size_hash); - if (!dht->ent_tab[i]) + if(!dht->ent_tab[i]) break; } - if (i != num_nodes) { + if(i != num_nodes) { errno = ENOMEM; - while (--i > 0) + while(--i > 0) free(dht->ent_tab[i]); free(dht); dht = 0; @@ -283,7 +289,7 @@ static void dht_free(struct dht *dht) { int i; - for (i = 0; i < dht->num_entries; i++) + for(i = 0; i < dht->num_entries; i++) free(dht->ent_tab[i]); free(dht); @@ -293,8 +299,8 @@ static void dht_free_v2(struct dht *dht) { int i; - for (i = 0; i < dht->num_entries; i++) { - if (dht->ent_tab[i]->bb_tab) { + for(i = 0; i < dht->num_entries; i++) { + if(dht->ent_tab[i]->bb_tab) { free(dht->ent_tab[i]->bb_tab); } free(dht->ent_tab[i]); @@ -305,26 +311,20 @@ static void dht_free_v2(struct dht *dht) static int dht_intersect(struct dht_entry *de, struct intv *itv) { - int i; - - if (de->i_virt.lb > itv->ub || de->i_virt.ub < itv->lb) - return 0; + int i; - for (i = 0; i < de->num_intv; i++) - if (intv_do_intersect(&de->i_tab[i], itv)) - return 1; + if(de->i_virt.lb > itv->ub || de->i_virt.ub < itv->lb) return 0; -} -static uint64_t ssd_get_max_dim(struct sspace *ss) -{ - return ss->max_dim; + for(i = 0; i < de->num_intv; i++) + if(intv_do_intersect(&de->i_tab[i], itv)) + return 1; + return 0; } -static int ssd_get_bpd(struct sspace *ss) -{ - return ss->bpd; -} +static uint64_t ssd_get_max_dim(struct sspace *ss) { return ss->max_dim; } + +static int ssd_get_bpd(struct sspace *ss) { return ss->bpd; } /* Hash the global geometric domain space to 1d index, and map a piece @@ -332,272 +332,275 @@ static int ssd_get_bpd(struct sspace *ss) */ static int dht_construct_hash(struct dht *dht, struct sspace *ssd) { - const uint64_t sn = bbox_volume(&dht->bb_glb_domain) / dht->num_entries; - struct intv *i_tab, intv; - struct dht_entry *de; - uint64_t len; - int num_intv, i, j; - int err = -ENOMEM; - - bbox_to_intv(&dht->bb_glb_domain, ssd->max_dim, ssd->bpd, - &i_tab, &num_intv); - - for (i = 0, j = 0; i < dht->num_entries; i++) { - len = sn; - - de = dht->ent_tab[i]; - de->rank = i; - de->i_tab = malloc(sizeof(struct intv) * num_intv); - if (!de->i_tab) - break; - - while (len > 0) { - if (intv_size(&i_tab[j]) > len) { - intv.lb = i_tab[j].lb; - intv.ub = intv.lb + len - 1; - i_tab[j].lb += len; - } - else { - intv = i_tab[j++]; - } - len -= intv_size(&intv); - de->i_tab[de->num_intv++] = intv; - } + const uint64_t sn = bbox_volume(&dht->bb_glb_domain) / dht->num_entries; + struct intv *i_tab, intv; + struct dht_entry *de; + uint64_t len; + int num_intv, i, j; + int err = -ENOMEM; + + bbox_to_intv(&dht->bb_glb_domain, ssd->max_dim, ssd->bpd, &i_tab, + &num_intv); - de->i_virt.lb = de->i_tab[0].lb; - de->i_virt.ub = de->i_tab[de->num_intv-1].ub; - de->i_tab = realloc(de->i_tab, sizeof(intv) * de->num_intv); - if (!de->i_tab) - break; + for(i = 0, j = 0; i < dht->num_entries; i++) { + len = sn; + de = dht->ent_tab[i]; + de->rank = i; + de->i_tab = malloc(sizeof(struct intv) * num_intv); + if(!de->i_tab) + break; + + while(len > 0) { + if(intv_size(&i_tab[j]) > len) { + intv.lb = i_tab[j].lb; + intv.ub = intv.lb + len - 1; + i_tab[j].lb += len; + } else { + intv = i_tab[j++]; + } + len -= intv_size(&intv); + de->i_tab[de->num_intv++] = intv; } - free(i_tab); + de->i_virt.lb = de->i_tab[0].lb; + de->i_virt.ub = de->i_tab[de->num_intv - 1].ub; + de->i_tab = realloc(de->i_tab, sizeof(intv) * de->num_intv); + if(!de->i_tab) + break; + } - if (i == dht->num_entries) - return 0; + free(i_tab); - fprintf(stderr, "'%s()': failed at entry %d.\n", __func__, i); - return err; + if(i == dht->num_entries) + return 0; + + fprintf(stderr, "'%s()': failed at entry %d.\n", __func__, i); + return err; } -static struct sspace *ssd_alloc_v1(const struct bbox *bb_domain, int num_nodes, int max_versions) +static struct sspace *ssd_alloc_v1(const struct bbox *bb_domain, int num_nodes, + int max_versions) { - struct sspace *ssd; - uint64_t max_dim; - int err = -ENOMEM; - - ssd = malloc(sizeof(*ssd)); - if (!ssd) - goto err_out; - memset(ssd, 0, sizeof(*ssd)); + struct sspace *ssd; + uint64_t max_dim; + int err = -ENOMEM; + ssd = malloc(sizeof(*ssd)); + if(!ssd) + goto err_out; + memset(ssd, 0, sizeof(*ssd)); - ssd->dht = dht_alloc(ssd, bb_domain, num_nodes, max_versions); - if (!ssd->dht) { - free(ssd); - goto err_out; - } + ssd->dht = dht_alloc(ssd, bb_domain, num_nodes, max_versions); + if(!ssd->dht) { + free(ssd); + goto err_out; + } - int i; - max_dim = bb_domain->ub.c[0]; - for(i = 1; i < bb_domain->num_dims; i++){ - max_dim = max(bb_domain->ub.c[i], max_dim); - } - if (max_dim == 0) max_dim = 1; // Note: max_dim as 0 would not work... + int i; + max_dim = bb_domain->ub.c[0]; + for(i = 1; i < bb_domain->num_dims; i++) { + max_dim = max(bb_domain->ub.c[i], max_dim); + } + if(max_dim == 0) + max_dim = 1; // Note: max_dim as 0 would not work... - ssd->max_dim = next_pow_2(max_dim); - ssd->bpd = compute_bits(ssd->max_dim); + ssd->max_dim = next_pow_2(max_dim); + ssd->bpd = compute_bits(ssd->max_dim); - err = dht_construct_hash(ssd->dht, ssd); - if (err < 0) { - dht_free(ssd->dht); - free(ssd); - goto err_out; - } + err = dht_construct_hash(ssd->dht, ssd); + if(err < 0) { + dht_free(ssd->dht); + free(ssd); + goto err_out; + } - ssd->hash_version = ssd_hash_version_v1; - return ssd; - err_out: - fprintf(stderr, "'%s()': failed with %d\n", __func__, err); - return NULL; + ssd->hash_version = ssd_hash_version_v1; + return ssd; +err_out: + fprintf(stderr, "'%s()': failed with %d\n", __func__, err); + return NULL; } static void ssd_free_v1(struct sspace *ssd) { - dht_free(ssd->dht); - free(ssd); - sh_free(); + dht_free(ssd->dht); + free(ssd); + sh_free(); } -static int ssd_hash_v1(struct sspace *ss, const struct bbox *bb, struct dht_entry *de_tab[]) +static int ssd_hash_v1(struct sspace *ss, const struct bbox *bb, + struct dht_entry *de_tab[]) { - struct intv *i_tab; - int i, k, n, num_nodes; + struct intv *i_tab; + int i, k, n, num_nodes; - num_nodes = sh_find(bb, de_tab); - if (num_nodes > 0) - /* This is great, I hit the cache. */ - return num_nodes; + num_nodes = sh_find(bb, de_tab); + if(num_nodes > 0) + /* This is great, I hit the cache. */ + return num_nodes; - num_nodes = 0; + num_nodes = 0; - bbox_to_intv2(bb, ss->max_dim, ss->bpd, &i_tab, &n); + bbox_to_intv2(bb, ss->max_dim, ss->bpd, &i_tab, &n); - for (k = 0; k < ss->dht->num_entries; k++){ - for (i = 0; i < n; i++) { - if (dht_intersect(ss->dht->ent_tab[k], &i_tab[i])) { - de_tab[num_nodes++] = ss->dht->ent_tab[k]; - break; - } - } + for(k = 0; k < ss->dht->num_entries; k++) { + for(i = 0; i < n; i++) { + if(dht_intersect(ss->dht->ent_tab[k], &i_tab[i])) { + de_tab[num_nodes++] = ss->dht->ent_tab[k]; + break; + } } + } - /* Cache the results for later use. */ - sh_add(bb, de_tab, num_nodes); + /* Cache the results for later use. */ + sh_add(bb, de_tab, num_nodes); - free(i_tab); - return num_nodes; + free(i_tab); + return num_nodes; } -struct sspace *ssd_alloc_v2(const struct bbox *bb_domain, int num_nodes, int max_versions) -{ - struct sspace *ssd = NULL; - int err = -ENOMEM; - int i, j, k; - int dim; - int nbits_max_dim = 0; - uint64_t max_dim = 0; - get_bbox_max_dim(bb_domain, &max_dim, &dim); - max_dim = next_pow_2_v2(max_dim); - nbits_max_dim = compute_bits_v2(max_dim); - - // decompose the global bbox - int num_divide_iteration = compute_bits_v2(next_pow_2_v2(num_nodes)); - struct bbox *bb, *b1, *b2; - struct queue q1, q2; - queue_init(&q1); - queue_init(&q2); - - bb = malloc(sizeof(struct bbox)); - memcpy(bb, bb_domain, sizeof(struct bbox)); - queue_enqueue(&q1, bb); - for (i = 0; i < num_divide_iteration; i++) { - struct queue *src_q, *dst_q; - if (!queue_is_empty(&q1) && queue_is_empty(&q2)) { - src_q = &q1; - dst_q = &q2; - } else if (!queue_is_empty(&q2) && queue_is_empty(&q1)) { - src_q = &q2; - dst_q = &q1; - } else { - printf("%s(): error, both q1 and q2 is (non)empty.\n", __func__); - } - while (!queue_is_empty(src_q)) { - uint64_t max_dim_size; - struct bbox b_tab[2]; - - bb = queue_dequeue(src_q); - get_bbox_max_dim_size(bb, &max_dim_size, &dim); - bbox_divide_in2_ondim(bb, b_tab, dim); - free(bb); bb = NULL; - b1 = malloc(sizeof(struct bbox)); - b2 = malloc(sizeof(struct bbox)); - *b1 = b_tab[0]; - *b2 = b_tab[1]; - queue_enqueue(dst_q, b1); - queue_enqueue(dst_q, b2); - } +struct sspace *ssd_alloc_v2(const struct bbox *bb_domain, int num_nodes, + int max_versions) +{ + struct sspace *ssd = NULL; + int err = -ENOMEM; + int i, j, k; + int dim; + int nbits_max_dim = 0; + uint64_t max_dim = 0; + get_bbox_max_dim(bb_domain, &max_dim, &dim); + max_dim = next_pow_2_v2(max_dim); + nbits_max_dim = compute_bits_v2(max_dim); + + // decompose the global bbox + int num_divide_iteration = compute_bits_v2(next_pow_2_v2(num_nodes)); + struct bbox *bb, *b1, *b2; + struct queue q1, q2; + queue_init(&q1); + queue_init(&q2); + + bb = malloc(sizeof(struct bbox)); + memcpy(bb, bb_domain, sizeof(struct bbox)); + queue_enqueue(&q1, bb); + for(i = 0; i < num_divide_iteration; i++) { + struct queue *src_q, *dst_q; + if(!queue_is_empty(&q1) && queue_is_empty(&q2)) { + src_q = &q1; + dst_q = &q2; + } else if(!queue_is_empty(&q2) && queue_is_empty(&q1)) { + src_q = &q2; + dst_q = &q1; + } else { + printf("%s(): error, both q1 and q2 is (non)empty.\n", __func__); } + while(!queue_is_empty(src_q)) { + uint64_t max_dim_size; + struct bbox b_tab[2]; - struct queue *q; - if (!queue_is_empty(&q1)) q = &q1; - else if (!queue_is_empty(&q2)) q = &q2; + bb = queue_dequeue(src_q); + get_bbox_max_dim_size(bb, &max_dim_size, &dim); + bbox_divide_in2_ondim(bb, b_tab, dim); + free(bb); + bb = NULL; + b1 = malloc(sizeof(struct bbox)); + b2 = malloc(sizeof(struct bbox)); + *b1 = b_tab[0]; + *b2 = b_tab[1]; + queue_enqueue(dst_q, b1); + queue_enqueue(dst_q, b2); + } + } - ssd = malloc(sizeof(*ssd)); - if (!ssd) - goto err_out; - memset(ssd, 0, sizeof(*ssd)); + struct queue *q; + if(!queue_is_empty(&q1)) + q = &q1; + else if(!queue_is_empty(&q2)) + q = &q2; - ssd->max_dim = max_dim; - ssd->bpd = nbits_max_dim; + ssd = malloc(sizeof(*ssd)); + if(!ssd) + goto err_out; + memset(ssd, 0, sizeof(*ssd)); - ssd->total_num_bbox = queue_size(q); - ssd->dht = dht_alloc(ssd, bb_domain, num_nodes, max_versions); - if (!ssd->dht) { - free(ssd); - goto err_out; - } - for (i = 0; i < num_nodes; i++) { - ssd->dht->ent_tab[i]->rank = i; - } + ssd->max_dim = max_dim; + ssd->bpd = nbits_max_dim; - int n = ceil(ssd->total_num_bbox*1.0 / ssd->dht->num_entries); - for (i = 0; i < ssd->dht->num_entries; i++) { - ssd->dht->ent_tab[i]->size_bb_tab = n; - ssd->dht->ent_tab[i]->bb_tab = malloc(sizeof(struct bbox)*n); - } + ssd->total_num_bbox = queue_size(q); + ssd->dht = dht_alloc(ssd, bb_domain, num_nodes, max_versions); + if(!ssd->dht) { + free(ssd); + goto err_out; + } + for(i = 0; i < num_nodes; i++) { + ssd->dht->ent_tab[i]->rank = i; + } - // simple round-robing mapping of decomposed bbox to dht entries - i = 0; - while (! queue_is_empty(q)) { - bb = queue_dequeue(q); - j = i++ % ssd->dht->num_entries; - k = ssd->dht->ent_tab[j]->num_bbox++; - ssd->dht->ent_tab[j]->bb_tab[k] = *bb; - free(bb); - } + int n = ceil(ssd->total_num_bbox * 1.0 / ssd->dht->num_entries); + for(i = 0; i < ssd->dht->num_entries; i++) { + ssd->dht->ent_tab[i]->size_bb_tab = n; + ssd->dht->ent_tab[i]->bb_tab = malloc(sizeof(struct bbox) * n); + } - ssd->hash_version = ssd_hash_version_v2; - return ssd; - err_out: - fprintf(stderr, "'%s()': failed with %d\n", __func__, err); - return NULL; + // simple round-robing mapping of decomposed bbox to dht entries + i = 0; + while(!queue_is_empty(q)) { + bb = queue_dequeue(q); + j = i++ % ssd->dht->num_entries; + k = ssd->dht->ent_tab[j]->num_bbox++; + ssd->dht->ent_tab[j]->bb_tab[k] = *bb; + free(bb); + } + + ssd->hash_version = ssd_hash_version_v2; + return ssd; +err_out: + fprintf(stderr, "'%s()': failed with %d\n", __func__, err); + return NULL; } void ssd_free_v2(struct sspace *ssd) { - dht_free_v2(ssd->dht); - free(ssd); - sh_free(); + dht_free_v2(ssd->dht); + free(ssd); + sh_free(); } -int ssd_hash_v2(struct sspace *ss, const struct bbox *bb, struct dht_entry *de_tab[]) +int ssd_hash_v2(struct sspace *ss, const struct bbox *bb, + struct dht_entry *de_tab[]) { - int i, j, num_nodes; + int i, j, num_nodes; - num_nodes = sh_find(bb, de_tab); - if (num_nodes > 0) - /* This is great, I hit the cache. */ - return num_nodes; + num_nodes = sh_find(bb, de_tab); + if(num_nodes > 0) + /* This is great, I hit the cache. */ + return num_nodes; - num_nodes = 0; - for (i = 0; i < ss->dht->num_entries; i++) { - for (j = 0; j < ss->dht->ent_tab[i]->num_bbox; j++) { - if (bbox_does_intersect(bb, &ss->dht->ent_tab[i]->bb_tab[j])) { - de_tab[num_nodes++] = ss->dht->ent_tab[i]; - break; - } + num_nodes = 0; + for(i = 0; i < ss->dht->num_entries; i++) { + for(j = 0; j < ss->dht->ent_tab[i]->num_bbox; j++) { + if(bbox_does_intersect(bb, &ss->dht->ent_tab[i]->bb_tab[j])) { + de_tab[num_nodes++] = ss->dht->ent_tab[i]; + break; } } + } - /* Cache the results for later use. */ - sh_add(bb, de_tab, num_nodes); + /* Cache the results for later use. */ + sh_add(bb, de_tab, num_nodes); - return num_nodes; + return num_nodes; } - - static void matrix_init(struct matrix *mat, enum storage_type st, - struct bbox *bb_glb, struct bbox *bb_loc, - void *pdata, size_t se) + struct bbox *bb_glb, struct bbox *bb_loc, void *pdata, + size_t se) { int i; int ndims = bb_glb->num_dims; memset(mat, 0, sizeof(struct matrix)); - for(i = 0; i < ndims; i++){ + for(i = 0; i < ndims; i++) { mat->dist[i] = bbox_dist(bb_glb, i); mat->mat_view.lb[i] = bb_loc->lb.c[i] - bb_glb->lb.c[i]; mat->mat_view.ub[i] = bb_loc->ub.c[i] - bb_glb->lb.c[i]; @@ -611,109 +614,137 @@ static void matrix_init(struct matrix *mat, enum storage_type st, static int matrix_copy(struct matrix *a, struct matrix *b) { - char *A = a->pdata; - char *B = b->pdata; + char *A = a->pdata; + char *B = b->pdata; - uint64_t a0, a1, a2, a3, a4, a5, a6, a7, a8, a9; - uint64_t aloc=0, aloc1=0, aloc2=0, aloc3=0, aloc4=0, aloc5=0, aloc6=0, aloc7=0, aloc8=0, aloc9=0; - uint64_t b0, b1, b2, b3, b4, b5, b6, b7, b8, b9; - uint64_t bloc=0, bloc1=0, bloc2=0, bloc3=0, bloc4=0, bloc5=0, bloc6=0, bloc7=0, bloc8=0, bloc9=0; + uint64_t a0, a1, a2, a3, a4, a5, a6, a7, a8, a9; + uint64_t aloc = 0, aloc1 = 0, aloc2 = 0, aloc3 = 0, aloc4 = 0, aloc5 = 0, + aloc6 = 0, aloc7 = 0, aloc8 = 0, aloc9 = 0; + uint64_t b0, b1, b2, b3, b4, b5, b6, b7, b8, b9; + uint64_t bloc = 0, bloc1 = 0, bloc2 = 0, bloc3 = 0, bloc4 = 0, bloc5 = 0, + bloc6 = 0, bloc7 = 0, bloc8 = 0, bloc9 = 0; uint64_t numelem; uint64_t num_copied_elem = 0; - switch(a->num_dims){ - case(1): - goto dim1; - break; - case(2): - goto dim2; - break; - case(3): - goto dim3; - break; - case(4): - goto dim4; - break; - case(5): - goto dim5; - break; - case(6): - goto dim6; - break; - case(7): - goto dim7; - break; - case(8): - goto dim8; - break; - case(9): - goto dim9; - break; - case(10): - goto dim10; - break; - default: - break; + switch(a->num_dims) { + case(1): + goto dim1; + break; + case(2): + goto dim2; + break; + case(3): + goto dim3; + break; + case(4): + goto dim4; + break; + case(5): + goto dim5; + break; + case(6): + goto dim6; + break; + case(7): + goto dim7; + break; + case(8): + goto dim8; + break; + case(9): + goto dim9; + break; + case(10): + goto dim10; + break; + default: + break; } - -dim10: for(a9 = a->mat_view.lb[9], b9 = b->mat_view.lb[9]; //TODO-Q - a9 <= a->mat_view.ub[9]; a9++, b9++){ - aloc9 = a9 * a->dist[8]; - bloc9 = a9 * b->dist[8]; -dim9: for(a8 = a->mat_view.lb[8], b8 = b->mat_view.lb[8]; //TODO-Q - a8 <= a->mat_view.ub[8]; a8++, b8++){ + +dim10: + for(a9 = a->mat_view.lb[9], b9 = b->mat_view.lb[9]; // TODO-Q + a9 <= a->mat_view.ub[9]; a9++, b9++) { + aloc9 = a9 * a->dist[8]; + bloc9 = a9 * b->dist[8]; + dim9: + for(a8 = a->mat_view.lb[8], b8 = b->mat_view.lb[8]; // TODO-Q + a8 <= a->mat_view.ub[8]; a8++, b8++) { aloc8 = (aloc9 + a8) * a->dist[7]; bloc8 = (bloc9 + b8) * b->dist[7]; -dim8: for(a7 = a->mat_view.lb[7], b7 = b->mat_view.lb[7]; //TODO-Q - a7 <= a->mat_view.ub[7]; a7++, b7++){ - aloc7 = (aloc8 + a7) * a->dist[6]; - bloc7 = (bloc8 + b7) * b->dist[6]; -dim7: for(a6 = a->mat_view.lb[6], b6 = b->mat_view.lb[6]; //TODO-Q - a6 <= a->mat_view.ub[6]; a6++, b6++){ - aloc6 = (aloc7 + a6) * a->dist[5]; - bloc6 = (bloc7 + b6) * b->dist[5]; -dim6: for(a5 = a->mat_view.lb[5], b5 = b->mat_view.lb[5]; //TODO-Q - a5 <= a->mat_view.ub[5]; a5++, b5++){ - aloc5 = (aloc6 + a5) * a->dist[4]; - bloc5 = (bloc6 + b5) * b->dist[4]; -dim5: for(a4 = a->mat_view.lb[4], b4 = b->mat_view.lb[4]; - a4 <= a->mat_view.ub[4]; a4++, b4++){ - aloc4 = (aloc5 + a4) * a->dist[3]; - bloc4 = (bloc5 + b4) * b->dist[3]; -dim4: for(a3 = a->mat_view.lb[3], b3 = b->mat_view.lb[3]; - a3 <= a->mat_view.ub[3]; a3++, b3++){ - aloc3 = (aloc4 + a3) * a->dist[2]; - bloc3 = (bloc4 + b3) * b->dist[2]; -dim3: for(a2 = a->mat_view.lb[2], b2 = b->mat_view.lb[2]; - a2 <= a->mat_view.ub[2]; a2++, b2++){ - aloc2 = (aloc3 + a2) * a->dist[1]; - bloc2 = (bloc3 + b2) * b->dist[1]; -dim2: for(a1 = a->mat_view.lb[1], b1 = b->mat_view.lb[1]; - a1 <= a->mat_view.ub[1]; a1++, b1++){ - aloc1 = (aloc2 + a1) * a->dist[0]; - bloc1 = (bloc2 + b1) * b->dist[0]; -dim1: numelem = (a->mat_view.ub[0] - a->mat_view.lb[0]) + 1; - aloc = aloc1 + a->mat_view.lb[0]; - bloc = bloc1 + b->mat_view.lb[0]; - memcpy(&A[aloc*a->size_elem], &B[bloc*a->size_elem], (a->size_elem * numelem)); - num_copied_elem += numelem; - if(a->num_dims == 1) return num_copied_elem; + dim8: + for(a7 = a->mat_view.lb[7], b7 = b->mat_view.lb[7]; // TODO-Q + a7 <= a->mat_view.ub[7]; a7++, b7++) { + aloc7 = (aloc8 + a7) * a->dist[6]; + bloc7 = (bloc8 + b7) * b->dist[6]; + dim7: + for(a6 = a->mat_view.lb[6], b6 = b->mat_view.lb[6]; // TODO-Q + a6 <= a->mat_view.ub[6]; a6++, b6++) { + aloc6 = (aloc7 + a6) * a->dist[5]; + bloc6 = (bloc7 + b6) * b->dist[5]; + dim6: + for(a5 = a->mat_view.lb[5], + b5 = b->mat_view.lb[5]; // TODO-Q + a5 <= a->mat_view.ub[5]; a5++, b5++) { + aloc5 = (aloc6 + a5) * a->dist[4]; + bloc5 = (bloc6 + b5) * b->dist[4]; + dim5: + for(a4 = a->mat_view.lb[4], b4 = b->mat_view.lb[4]; + a4 <= a->mat_view.ub[4]; a4++, b4++) { + aloc4 = (aloc5 + a4) * a->dist[3]; + bloc4 = (bloc5 + b4) * b->dist[3]; + dim4: + for(a3 = a->mat_view.lb[3], b3 = b->mat_view.lb[3]; + a3 <= a->mat_view.ub[3]; a3++, b3++) { + aloc3 = (aloc4 + a3) * a->dist[2]; + bloc3 = (bloc4 + b3) * b->dist[2]; + dim3: + for(a2 = a->mat_view.lb[2], + b2 = b->mat_view.lb[2]; + a2 <= a->mat_view.ub[2]; a2++, b2++) { + aloc2 = (aloc3 + a2) * a->dist[1]; + bloc2 = (bloc3 + b2) * b->dist[1]; + dim2: + for(a1 = a->mat_view.lb[1], + b1 = b->mat_view.lb[1]; + a1 <= a->mat_view.ub[1]; a1++, b1++) { + aloc1 = (aloc2 + a1) * a->dist[0]; + bloc1 = (bloc2 + b1) * b->dist[0]; + dim1: + numelem = (a->mat_view.ub[0] - + a->mat_view.lb[0]) + + 1; + aloc = aloc1 + a->mat_view.lb[0]; + bloc = bloc1 + b->mat_view.lb[0]; + memcpy(&A[aloc * a->size_elem], + &B[bloc * a->size_elem], + (a->size_elem * numelem)); + num_copied_elem += numelem; + if(a->num_dims == 1) + return num_copied_elem; + } + if(a->num_dims == 2) + return num_copied_elem; + } + if(a->num_dims == 3) + return num_copied_elem; + } + if(a->num_dims == 4) + return num_copied_elem; + } + if(a->num_dims == 5) + return num_copied_elem; + } + if(a->num_dims == 6) + return num_copied_elem; } - if(a->num_dims == 2) return num_copied_elem; + if(a->num_dims == 7) + return num_copied_elem; } - if(a->num_dims == 3) return num_copied_elem; + if(a->num_dims == 8) + return num_copied_elem; } - if(a->num_dims == 4) return num_copied_elem; - } - if(a->num_dims == 5) return num_copied_elem; - } - if(a->num_dims == 6) return num_copied_elem; - } - if(a->num_dims == 7) return num_copied_elem; - } - if(a->num_dims == 8) return num_copied_elem; - } - if(a->num_dims == 9) return num_copied_elem; + if(a->num_dims == 9) + return num_copied_elem; } } @@ -723,77 +754,78 @@ char *obj_desc_sprint(obj_descriptor *odsc) int nb; str = alloc_sprintf("obj_descriptor = {\n" - "\t.name = %s,\n" - "\t.owner = %s,\n" - "\t.version = %d,\n" - "\t.bb = ", odsc->name, odsc->owner, odsc->version); + "\t.name = %s,\n" + "\t.owner = %s,\n" + "\t.version = %d,\n" + "\t.bb = ", + odsc->name, odsc->owner, odsc->version); str = str_append_const(str_append(str, bbox_sprint(&odsc->bb)), "}\n"); return str; } /* -*/ + */ int ssd_copy(struct obj_data *to_obj, struct obj_data *from_obj) { - struct matrix to_mat, from_mat; - struct bbox bbcom; - int copied_elems = 0; + struct matrix to_mat, from_mat; + struct bbox bbcom; + int copied_elems = 0; - bbox_intersect(&to_obj->obj_desc.bb, &from_obj->obj_desc.bb, &bbcom); + bbox_intersect(&to_obj->obj_desc.bb, &from_obj->obj_desc.bb, &bbcom); - matrix_init(&from_mat, from_obj->obj_desc.st, - &from_obj->obj_desc.bb, &bbcom, - from_obj->data, from_obj->obj_desc.size); + matrix_init(&from_mat, from_obj->obj_desc.st, &from_obj->obj_desc.bb, + &bbcom, from_obj->data, from_obj->obj_desc.size); - matrix_init(&to_mat, to_obj->obj_desc.st, - &to_obj->obj_desc.bb, &bbcom, - to_obj->data, to_obj->obj_desc.size); + matrix_init(&to_mat, to_obj->obj_desc.st, &to_obj->obj_desc.bb, &bbcom, + to_obj->data, to_obj->obj_desc.size); - copied_elems = matrix_copy(&to_mat, &from_mat); - return copied_elems; + copied_elems = matrix_copy(&to_mat, &from_mat); + return copied_elems; } - /* Allocate and init the local storage structure. */ ss_storage *ls_alloc(int max_versions) { - ss_storage *ls = 0; - int i; + ss_storage *ls = 0; + int i; - ls = malloc(sizeof(*ls) + sizeof(struct list_head) * max_versions); - if (!ls) { - errno = ENOMEM; - return ls; - } + ls = malloc(sizeof(*ls) + sizeof(struct list_head) * max_versions); + if(!ls) { + errno = ENOMEM; + return ls; + } - memset(ls, 0, sizeof(*ls)); - for (i = 0; i < max_versions; i++) - INIT_LIST_HEAD(&ls->obj_hash[i]); - ls->size_hash = max_versions; + memset(ls, 0, sizeof(*ls)); + for(i = 0; i < max_versions; i++) + INIT_LIST_HEAD(&ls->obj_hash[i]); + ls->size_hash = max_versions; - return ls; + return ls; } void ls_free(ss_storage *ls) { - if (!ls) return; + if(!ls) + return; struct obj_data *od, *t; struct list_head *list; int i; - for (i = 0; i < ls->size_hash; i++) { + for(i = 0; i < ls->size_hash; i++) { list = &ls->obj_hash[i]; - list_for_each_entry_safe(od, t, list, struct obj_data, obj_entry ) { + list_for_each_entry_safe(od, t, list, struct obj_data, obj_entry) + { ls_remove(ls, od); obj_data_free(od); } } - if (ls->num_obj != 0) { - fprintf(stderr, "%s(): ERROR ls->num_obj is %d not 0\n", __func__, ls->num_obj); + if(ls->num_obj != 0) { + fprintf(stderr, "%s(): ERROR ls->num_obj is %d not 0\n", __func__, + ls->num_obj); } free(ls); } @@ -803,65 +835,65 @@ void ls_free(ss_storage *ls) */ void ls_add_obj(ss_storage *ls, struct obj_data *od) { - int index; - struct list_head *bin; - struct obj_data *od_existing; - - //ABT_rwlock_create(&od->lock); - //ABT_rwlock_wrlock(&od->lock); - - od_existing = ls_find_no_version(ls, &od->obj_desc); - if (od_existing) { - od_existing->f_free = 1; - if (od_existing->refcnt == 0) { - ls_remove(ls, od_existing); - obj_data_free(od_existing); - } - else { - fprintf(stderr, "'%s()': object eviction delayed.\n", __func__); - } + int index; + struct list_head *bin; + struct obj_data *od_existing; + + // ABT_rwlock_create(&od->lock); + // ABT_rwlock_wrlock(&od->lock); + + od_existing = ls_find_no_version(ls, &od->obj_desc); + if(od_existing) { + od_existing->f_free = 1; + if(od_existing->refcnt == 0) { + ls_remove(ls, od_existing); + obj_data_free(od_existing); + } else { + fprintf(stderr, "'%s()': object eviction delayed.\n", __func__); } - index = od->obj_desc.version % ls->size_hash; - bin = &ls->obj_hash[index]; + } + index = od->obj_desc.version % ls->size_hash; + bin = &ls->obj_hash[index]; - /* NOTE: new object comes first in the list. */ - list_add(&od->obj_entry, bin); - ls->num_obj++; - //ABT_rwlock_unlock(&od->lock); + /* NOTE: new object comes first in the list. */ + list_add(&od->obj_entry, bin); + ls->num_obj++; + // ABT_rwlock_unlock(&od->lock); } -struct obj_data* ls_lookup(ss_storage *ls, char *name) +struct obj_data *ls_lookup(ss_storage *ls, char *name) { - struct obj_data *od; - struct list_head *list; - int i; + struct obj_data *od; + struct list_head *list; + int i; - for (i = 0; i < ls->size_hash; i++) { - list = &ls->obj_hash[i]; + for(i = 0; i < ls->size_hash; i++) { + list = &ls->obj_hash[i]; - list_for_each_entry(od, list, struct obj_data, obj_entry ) { - if (strcmp(od->obj_desc.name, name) == 0) - return od; - } + list_for_each_entry(od, list, struct obj_data, obj_entry) + { + if(strcmp(od->obj_desc.name, name) == 0) + return od; } + } - return NULL; + return NULL; } void ls_remove(ss_storage *ls, struct obj_data *od) { - list_del(&od->obj_entry); - ls->num_obj--; + list_del(&od->obj_entry); + ls->num_obj--; } void ls_try_remove_free(ss_storage *ls, struct obj_data *od) { - /* Note: we assume the object data is allocated with - obj_data_alloc(), i.e., the data follows the structure. */ - if (od->refcnt == 0) { - ls_remove(ls, od); - obj_data_free(od); - } + /* Note: we assume the object data is allocated with + obj_data_alloc(), i.e., the data follows the structure. */ + if(od->refcnt == 0) { + ls_remove(ls, od); + obj_data_free(od); + } } /* @@ -870,40 +902,42 @@ void ls_try_remove_free(ss_storage *ls, struct obj_data *od) */ struct obj_data *ls_find(ss_storage *ls, obj_descriptor *odsc) { - struct obj_data *od; - struct list_head *list; - int index; + struct obj_data *od; + struct list_head *list; + int index; - index = odsc->version % ls->size_hash; - list = &ls->obj_hash[index]; + index = odsc->version % ls->size_hash; + list = &ls->obj_hash[index]; - list_for_each_entry(od, list, struct obj_data, obj_entry) { - if (obj_desc_equals_intersect(odsc, &od->obj_desc)) - return od; - } + list_for_each_entry(od, list, struct obj_data, obj_entry) + { + if(obj_desc_equals_intersect(odsc, &od->obj_desc)) + return od; + } - return NULL; + return NULL; } /* - Find list of object_desriptors in the local storage that has the same name and - version with the object descriptor 'odsc'. + Find list of object_desriptors in the local storage that has the same + name and version with the object descriptor 'odsc'. */ int ls_find_ods(ss_storage *ls, obj_descriptor *odsc, struct obj_data **od_tab) { - struct obj_data *od; - struct list_head *list; - int index; - int num_odsc = 0; + struct obj_data *od; + struct list_head *list; + int index; + int num_odsc = 0; - index = odsc->version % ls->size_hash; - list = &ls->obj_hash[index]; - list_for_each_entry(od, list, struct obj_data, obj_entry) { - if (obj_desc_equals_intersect(odsc, &od->obj_desc)){ - od_tab[num_odsc++] = od; - } + index = odsc->version % ls->size_hash; + list = &ls->obj_hash[index]; + list_for_each_entry(od, list, struct obj_data, obj_entry) + { + if(obj_desc_equals_intersect(odsc, &od->obj_desc)) { + od_tab[num_odsc++] = od; } - return num_odsc; + } + return num_odsc; } /* @@ -911,22 +945,22 @@ int ls_find_ods(ss_storage *ls, obj_descriptor *odsc, struct obj_data **od_tab) bin, and that has the same name and object descriptor, but may have different version. */ -struct obj_data * -ls_find_no_version(ss_storage *ls, obj_descriptor *odsc) +struct obj_data *ls_find_no_version(ss_storage *ls, obj_descriptor *odsc) { - struct obj_data *od; - struct list_head *list; - int index; + struct obj_data *od; + struct list_head *list; + int index; - index = odsc->version % ls->size_hash; - list = &ls->obj_hash[index]; + index = odsc->version % ls->size_hash; + list = &ls->obj_hash[index]; - list_for_each_entry(od, list, struct obj_data, obj_entry) { - if (obj_desc_by_name_intersect(odsc, &od->obj_desc)) - return od; - } + list_for_each_entry(od, list, struct obj_data, obj_entry) + { + if(obj_desc_by_name_intersect(odsc, &od->obj_desc)) + return od; + } - return NULL; + return NULL; } /* @@ -954,11 +988,10 @@ struct lock_data *create_lock(struct list_head *lock_list, char* name) } */ - -#define ALIGN_ADDR_QUAD_BYTES(a) \ - unsigned long _a = (unsigned long) (a); \ - _a = (_a + 7) & ~7; \ - (a) = (void *) _a; +#define ALIGN_ADDR_QUAD_BYTES(a) \ + unsigned long _a = (unsigned long)(a); \ + _a = (_a + 7) & ~7; \ + (a) = (void *)_a; /* Allocate space for an obj_data structure and the data. */ @@ -966,108 +999,101 @@ struct obj_data *obj_data_alloc(obj_descriptor *odsc) { struct obj_data *od = 0; - od = malloc(sizeof(*od)); - if (!od){ + od = malloc(sizeof(*od)); + if(!od) { fprintf(stderr, "Malloc od error\n"); - return NULL; + return NULL; } - memset(od, 0, sizeof(*od)); + memset(od, 0, sizeof(*od)); - int size= obj_data_size(odsc); - od->data= malloc(size); - if (!od->data) { + int size = obj_data_size(odsc); + od->data = malloc(size); + if(!od->data) { fprintf(stderr, "Malloc od_data error\n"); free(od); return NULL; } - ALIGN_ADDR_QUAD_BYTES(od->data); - od->obj_desc = *odsc; + ALIGN_ADDR_QUAD_BYTES(od->data); + od->obj_desc = *odsc; return od; } -/* +/* Allocate space for the obj_data struct only; space for data is externally allocated. */ -struct obj_data * obj_data_alloc_no_data(obj_descriptor *odsc, void *data) +struct obj_data *obj_data_alloc_no_data(obj_descriptor *odsc, void *data) { - struct obj_data *od; + struct obj_data *od; - od = malloc(sizeof(*od)); - if (!od) - return NULL; - memset(od, 0, sizeof(*od)); + od = malloc(sizeof(*od)); + if(!od) + return NULL; + memset(od, 0, sizeof(*od)); - od->obj_desc = *odsc; - od->data = data; + od->obj_desc = *odsc; + od->data = data; - return od; + return od; } struct obj_data *obj_data_alloc_with_data(obj_descriptor *odsc, void *data) { - struct obj_data *od = obj_data_alloc(odsc); - if (!od) - return NULL; + struct obj_data *od = obj_data_alloc(odsc); + if(!od) + return NULL; - memcpy(od->data, data, obj_data_size(odsc)); - //TODO: what about the descriptor ? + memcpy(od->data, data, obj_data_size(odsc)); + // TODO: what about the descriptor ? - return od; + return od; } - - void obj_data_free(struct obj_data *od) { - if(od){ - if(od->data){ - free(od->data); + if(od) { + if(od->data) { + free(od->data); } - free(od); + free(od); } } - - uint64_t obj_data_size(obj_descriptor *obj_desc) { return obj_desc->size * bbox_volume(&obj_desc->bb); } - int obj_desc_equals_no_owner(const obj_descriptor *odsc1, - const obj_descriptor *odsc2) + const obj_descriptor *odsc2) { - if (odsc1->version == odsc2->version && - strcmp(odsc1->name, odsc2->name) == 0 && - bbox_equals(&odsc1->bb, &odsc2->bb)) - return 1; - return 0; + if(odsc1->version == odsc2->version && + strcmp(odsc1->name, odsc2->name) == 0 && + bbox_equals(&odsc1->bb, &odsc2->bb)) + return 1; + return 0; } -int obj_desc_equals(obj_descriptor *odsc1, - obj_descriptor *odsc2) +int obj_desc_equals(obj_descriptor *odsc1, obj_descriptor *odsc2) { - if (odsc1->owner == odsc2->owner && - bbox_equals(&odsc1->bb, &odsc2->bb)) - return 1; - else return 0; + if(odsc1->owner == odsc2->owner && bbox_equals(&odsc1->bb, &odsc2->bb)) + return 1; + else + return 0; } /* * Test if two object descriptors have the same name and versions and * their bounding boxes intersect. * */ -int obj_desc_equals_intersect(obj_descriptor *odsc1, - obj_descriptor *odsc2) +int obj_desc_equals_intersect(obj_descriptor *odsc1, obj_descriptor *odsc2) { - if (strcmp(odsc1->name, odsc2->name) == 0 && - odsc1->version == odsc2->version && - bbox_does_intersect(&odsc1->bb, &odsc2->bb)) - return 1; - return 0; + if(strcmp(odsc1->name, odsc2->name) == 0 && + odsc1->version == odsc2->version && + bbox_does_intersect(&odsc1->bb, &odsc2->bb)) + return 1; + return 0; } /* @@ -1075,12 +1101,12 @@ int obj_desc_equals_intersect(obj_descriptor *odsc1, * boxes intersect. * */ int obj_desc_by_name_intersect(const obj_descriptor *odsc1, - const obj_descriptor *odsc2) + const obj_descriptor *odsc2) { - if (strcmp(odsc1->name, odsc2->name) == 0 && - bbox_does_intersect(&odsc1->bb, &odsc2->bb)) - return 1; - return 0; + if(strcmp(odsc1->name, odsc2->name) == 0 && + bbox_does_intersect(&odsc1->bb, &odsc2->bb)) + return 1; + return 0; } /* @@ -1096,8 +1122,9 @@ int obj_desc_by_name_intersect(const obj_descriptor *odsc1, /* Allocate the shared space structure. */ -struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, int max_versions, - enum sspace_hash_version hash_version) +struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, + int max_versions, + enum sspace_hash_version hash_version) { struct sspace *ss = NULL; @@ -1109,13 +1136,12 @@ struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, int max_ve tm_st = timer_read(&tm); #endif - #ifdef DEBUG char *str = bbox_sprint(bb_domain); fprintf(stderr, "%s: allocating new shared space %s\n", __func__, str); #endif - switch (hash_version) { + switch(hash_version) { case ssd_hash_version_v1: ss = ssd_alloc_v1(bb_domain, num_nodes, max_versions); break; @@ -1124,13 +1150,14 @@ struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, int max_ve break; default: fprintf(stderr, "%s(): ERROR unknown shared space hash version %u\n", - __func__, hash_version); + __func__, hash_version); break; } -#ifdef TIMING_SSD +#ifdef TIMING_SSD tm_end = timer_read(&tm); - fprintf(stderr, "%s(): hash_version v%u time %lf seconds\n", __func__, hash_version, tm_end-tm_st); + fprintf(stderr, "%s(): hash_version v%u time %lf seconds\n", __func__, + hash_version, tm_end - tm_st); #endif return ss; @@ -1138,7 +1165,7 @@ struct sspace *ssd_alloc(const struct bbox *bb_domain, int num_nodes, int max_ve void ssd_free(struct sspace *ss) { - switch (ss->hash_version) { + switch(ss->hash_version) { case ssd_hash_version_v1: ssd_free_v1(ss); break; @@ -1147,7 +1174,7 @@ void ssd_free(struct sspace *ss) break; default: fprintf(stderr, "%s(): ERROR unknown shared space hash version %u\n", - __func__, ss->hash_version); + __func__, ss->hash_version); break; } } @@ -1158,7 +1185,7 @@ long ssh_hash_elem_count_v1(struct sspace *ss, const struct bbox *bb) long overlap, num_elem; int i, n; - //TODO: cache results + // TODO: cache results bbox_to_intv2(bb, ss->max_dim, ss->bpd, &i_tab, &n); @@ -1178,7 +1205,7 @@ long ssh_hash_elem_count_v1(struct sspace *ss, const struct bbox *bb) } } - return(num_elem); + return (num_elem); } long ssh_hash_elem_count_v2(struct sspace *ss, const struct bbox *bb) @@ -1188,7 +1215,7 @@ long ssh_hash_elem_count_v2(struct sspace *ss, const struct bbox *bb) struct bbox isect; int i; - //TODO: cache results + // TODO: cache results num_elem = 0; for(i = 0; i < ss->ent_self->num_bbox; i++) { @@ -1199,10 +1226,9 @@ long ssh_hash_elem_count_v2(struct sspace *ss, const struct bbox *bb) } } - return(num_elem); + return (num_elem); } - /* Get the number of elements of a bounding box 'bb' that hash to the local dht @@ -1220,19 +1246,20 @@ long ssh_hash_elem_count(struct sspace *ss, const struct bbox *bb) break; default: fprintf(stderr, "%s(): ERROR unknown shared space hash version %u\n", - __func__, ss->hash_version); + __func__, ss->hash_version); ret = 0; break; } - return(ret); + return (ret); } /* Hash a bounding box 'bb' to the hash entries in dht; fill in the entries in the de_tab and return the number of entries. */ -int ssd_hash(struct sspace *ss, const struct bbox *bb, struct dht_entry *de_tab[]) +int ssd_hash(struct sspace *ss, const struct bbox *bb, + struct dht_entry *de_tab[]) { #ifdef TIMING_SSD struct timer tm; @@ -1244,7 +1271,7 @@ int ssd_hash(struct sspace *ss, const struct bbox *bb, struct dht_entry *de_tab[ int ret; - switch (ss->hash_version) { + switch(ss->hash_version) { case ssd_hash_version_v1: ret = ssd_hash_v1(ss, bb, de_tab); break; @@ -1253,14 +1280,15 @@ int ssd_hash(struct sspace *ss, const struct bbox *bb, struct dht_entry *de_tab[ break; default: fprintf(stderr, "%s(): ERROR unknown shared space hash version %u\n", - __func__, ss->hash_version); + __func__, ss->hash_version); ret = 0; break; } #ifdef TIMING_SSD tm_end = timer_read(&tm); - fprintf(stderr, "%s(): hash_version v%u time %lf seconds\n", __func__, ss->hash_version, tm_end-tm_st); + fprintf(stderr, "%s(): hash_version v%u time %lf seconds\n", __func__, + ss->hash_version, tm_end - tm_st); #endif return ret; } @@ -1278,34 +1306,36 @@ int ssd_init(struct sspace *ssd, int rank) int ssd_filter(struct obj_data *from, obj_descriptor *odsc, double *dval) { - //TODO: search the matrix to find the min - static int n = 1; + // TODO: search the matrix to find the min + static int n = 1; - *dval = 2.0 * n; - n++; + *dval = 2.0 * n; + n++; - return 0; + return 0; } /* Test if the 'odsc' matches any object descriptor in a DHT entry by name and coordinates, but not version, and return the matching index. */ -static struct obj_desc_list * -dht_find_match(const struct dht_entry *de, const obj_descriptor *odsc) +static struct obj_desc_list *dht_find_match(const struct dht_entry *de, + const obj_descriptor *odsc) { struct obj_desc_list *odscl; - int n; + int n; // TODO: delete this (just an assertion for proper behaviour). - if (odsc->version == (unsigned int) -1) { - fprintf(stderr, "'%s()': version on object descriptor is not set!!!\n", - __func__); + if(odsc->version == (unsigned int)-1) { + fprintf(stderr, "'%s()': version on object descriptor is not set!!!\n", + __func__); return 0; } n = odsc->version % de->odsc_size; - list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, odsc_entry) { + list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, + odsc_entry) + { if(obj_desc_by_name_intersect(odsc, &odscl->odsc)) return odscl; } @@ -1313,30 +1343,34 @@ dht_find_match(const struct dht_entry *de, const obj_descriptor *odsc) return 0; } -static obj_descriptor *dht_find_exact(const struct dht_entry *de, const obj_descriptor *odsc) +static obj_descriptor *dht_find_exact(const struct dht_entry *de, + const obj_descriptor *odsc) { struct obj_desc_list *odscl; int n; n = odsc->version % de->odsc_size; - list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, odsc_entry) { - if(obj_desc_equals_no_owner(odsc, &odscl->odsc)) { - return(&odscl->odsc); - } + list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, + odsc_entry) + { + if(obj_desc_equals_no_owner(odsc, &odscl->odsc)) { + return (&odscl->odsc); + } } - return(NULL); + return (NULL); } #define array_resize(a, n) a = realloc(a, sizeof(*a) * (n)) /*` * subscribe to a certain number of elements, remaining, of an object descriptor - * q_odsc. When dht updates are added, they will be checked against subscriptions - * created by this function. Each incoming update that overlaps with q_odsc will - * decrement sub.remaining by the size of the overlap and increment pub_count by - * one. The general idea is that once enough elements that overlap with the query - * have been counted, the query must be fully satisfiable. + * q_odsc. When dht updates are added, they will be checked against + * subscriptions created by this function. Each incoming update that overlaps + * with q_odsc will decrement sub.remaining by the size of the overlap and + * increment pub_count by one. The general idea is that once enough elements + * that overlap with the query have been counted, the query must be fully + * satisfiable. * * Updates and signals are coming from dht_add_entry * @@ -1344,8 +1378,9 @@ static obj_descriptor *dht_find_exact(const struct dht_entry *de, const obj_desc * subscription should fail and that failure should be propagated. * */ -void dht_local_subscribe(struct dht_entry *de, obj_descriptor *q_odsc, - obj_descriptor ***odsc_tab, int *tab_entries, long remaining, int timeout) +void dht_local_subscribe(struct dht_entry *de, obj_descriptor *q_odsc, + obj_descriptor ***odsc_tab, int *tab_entries, + long remaining, int timeout) { struct dht_sub_list_entry sub; obj_descriptor **odsc_tab_pos; @@ -1355,24 +1390,26 @@ void dht_local_subscribe(struct dht_entry *de, obj_descriptor *q_odsc, sub.odsc = q_odsc; sub.remaining = remaining; sub.pub_count = 0; - INIT_LIST_HEAD(&sub.recv_odsc); - + INIT_LIST_HEAD(&sub.recv_odsc); + list_add(&sub.entry, &de->dht_subs[n]); - + do { ABT_cond_wait(de->hash_cond[n], de->hash_mutex[n]); - }while(sub.remaining > 0); + } while(sub.remaining > 0); - *odsc_tab = realloc(*odsc_tab,sizeof(**odsc_tab) * (*tab_entries + sub.pub_count)); + *odsc_tab = + realloc(*odsc_tab, sizeof(**odsc_tab) * (*tab_entries + sub.pub_count)); odsc_tab_pos = &(*odsc_tab)[*tab_entries]; - list_for_each_entry_safe(odscl, tmp, &sub.recv_odsc, struct obj_desc_ptr_list, odsc_entry) { + list_for_each_entry_safe(odscl, tmp, &sub.recv_odsc, + struct obj_desc_ptr_list, odsc_entry) + { *odsc_tab_pos = odscl->odsc; odsc_tab_pos++; list_del(&odscl->odsc_entry); free(odscl); } *tab_entries += sub.pub_count; - } int dht_update_owner(struct dht_entry *de, obj_descriptor *odsc) @@ -1381,13 +1418,15 @@ int dht_update_owner(struct dht_entry *de, obj_descriptor *odsc) old_odsc = dht_find_exact(de, odsc); if(!old_odsc) { - fprintf(stderr, "ERROR: (%s): no matching object found when doing update. Object being updated is %s\n", __func__, obj_desc_sprint(odsc)); - return(-ENOENT); + fprintf(stderr, + "ERROR: (%s): no matching object found when doing update. " + "Object being updated is %s\n", + __func__, obj_desc_sprint(odsc)); + return (-ENOENT); } - strcpy(old_odsc->owner, odsc->owner); + strcpy(old_odsc->owner, odsc->owner); return 0; - } int dht_add_entry(struct dht_entry *de, obj_descriptor *odsc) @@ -1400,13 +1439,18 @@ int dht_add_entry(struct dht_entry *de, obj_descriptor *odsc) int n, err = -ENOMEM; odscl = dht_find_match(de, odsc); - if (odscl) { - /* There is allready a descriptor with a different - version in the DHT, so I will overwrite it. */ + if(odscl) { + /* There is allready a descriptor with a different + version in the DHT, so I will overwrite it. */ if(odscl->odsc.version == odsc->version) { - fprintf(stderr, "WARNING: the server has detected an overlapping put (same version and some common elements with a previous put. DataSpaces storage is intended to be immutable, and the behavior of updates is undefined. Proceed at your own risk...\n"); + fprintf(stderr, "WARNING: the server has detected an overlapping " + "put (same version and some common elements with a " + "previous put. DataSpaces storage is intended to " + "be immutable, and the behavior of updates is " + "undefined. Proceed at your own risk...\n"); fprintf(stderr, " New put is: \n%s\n", obj_desc_sprint(odsc)); - fprintf(stderr, " But found existing: \n%s\n", obj_desc_sprint(&odscl->odsc)); + fprintf(stderr, " But found existing: \n%s\n", + obj_desc_sprint(&odscl->odsc)); } memcpy(&odscl->odsc, odsc, sizeof(*odsc)); return 0; @@ -1414,7 +1458,7 @@ int dht_add_entry(struct dht_entry *de, obj_descriptor *odsc) n = odsc->version % de->odsc_size; odscl = malloc(sizeof(*odscl)); - if (!odscl) + if(!odscl) return err; memcpy(&odscl->odsc, odsc, sizeof(*odsc)); @@ -1422,10 +1466,12 @@ int dht_add_entry(struct dht_entry *de, obj_descriptor *odsc) de->odsc_num++; ABT_mutex_lock(de->hash_mutex[n]); - list_for_each_entry_safe(sub, tmp, &de->dht_subs[n], struct dht_sub_list_entry, entry) { + list_for_each_entry_safe(sub, tmp, &de->dht_subs[n], + struct dht_sub_list_entry, entry) + { if(bbox_does_intersect(&odsc->bb, &sub->odsc->bb)) { sub_odscl = malloc(sizeof(*sub_odscl)); - sub_odscl->odsc = &odscl->odsc; + sub_odscl->odsc = &odscl->odsc; list_add(&sub_odscl->odsc_entry, &sub->recv_odsc); sub->pub_count++; @@ -1450,8 +1496,8 @@ int dht_add_entry(struct dht_entry *de, obj_descriptor *odsc) from dht entry 'de'; find all descriptor from 'de' and return their number and references . */ -int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, - obj_descriptor **odsc_tab[], int timeout) +int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, + obj_descriptor **odsc_tab[], int timeout) { int n, num_odsc = 0; long num_elem; @@ -1464,8 +1510,10 @@ int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, num_elem = ssh_hash_elem_count(de->ss, &q_odsc->bb); ABT_mutex_lock(de->hash_mutex[n]); } - list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, odsc_entry) { - if (obj_desc_equals_intersect(&odscl->odsc, q_odsc)) { + list_for_each_entry(odscl, &de->odsc_hash[n], struct obj_desc_list, + odsc_entry) + { + if(obj_desc_equals_intersect(&odscl->odsc, q_odsc)) { (*odsc_tab)[num_odsc++] = &odscl->odsc; if(sub) { bbox_intersect(&q_odsc->bb, &odscl->odsc.bb, &isect); @@ -1475,7 +1523,8 @@ int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, } if(sub) { if(num_elem > 0) { - dht_local_subscribe(de, q_odsc, odsc_tab, &num_odsc, num_elem, timeout); + dht_local_subscribe(de, q_odsc, odsc_tab, &num_odsc, num_elem, + timeout); } ABT_mutex_unlock(de->hash_mutex[n]); } @@ -1486,41 +1535,47 @@ int dht_find_entry_all(struct dht_entry *de, obj_descriptor *q_odsc, /* List the available versions of a data object. */ -int dht_find_versions(struct dht_entry *de, obj_descriptor *q_odsc, int odsc_vers[]) +int dht_find_versions(struct dht_entry *de, obj_descriptor *q_odsc, + int odsc_vers[]) { struct obj_desc_list *odscl; int i, n = 0; - for (i = 0; i < de->odsc_size; i++) { - list_for_each_entry(odscl, &de->odsc_hash[i], struct obj_desc_list, odsc_entry) - if (obj_desc_by_name_intersect(&odscl->odsc, q_odsc)) { - odsc_vers[n++] = odscl->odsc.version; - break; /* Break the list_for_each_entry loop. */ - } + for(i = 0; i < de->odsc_size; i++) { + list_for_each_entry( + odscl, &de->odsc_hash[i], struct obj_desc_list, + odsc_entry) if(obj_desc_by_name_intersect(&odscl->odsc, q_odsc)) + { + odsc_vers[n++] = odscl->odsc.version; + break; /* Break the list_for_each_entry loop. */ + } } return n; } void copy_global_dimension(struct global_dimension *l, int ndim, - const uint64_t *gdim) + const uint64_t *gdim) { int i; l->ndim = ndim; memset(&l->sizes, 0, sizeof(struct coord)); - for (i = 0; i < ndim; i++) { + for(i = 0; i < ndim; i++) { l->sizes.c[i] = gdim[i]; - } + } } void init_gdim_list(struct list_head *gdim_list) { - if (!gdim_list) return; + if(!gdim_list) + return; INIT_LIST_HEAD(gdim_list); } -void free_gdim_list(struct list_head *gdim_list) { - if (!gdim_list) return; +void free_gdim_list(struct list_head *gdim_list) +{ + if(!gdim_list) + return; int cnt = 0; struct gdim_list_entry *e, *t; list_for_each_entry_safe(e, t, gdim_list, struct gdim_list_entry, entry) @@ -1532,32 +1587,35 @@ void free_gdim_list(struct list_head *gdim_list) { } #ifdef DEBUG - fprintf(stderr, "%s(): number of user-defined global dimension is %d\n", __func__, cnt); + fprintf(stderr, "%s(): number of user-defined global dimension is %d\n", + __func__, cnt); #endif -} +} -struct gdim_list_entry* lookup_gdim_list(struct list_head *gdim_list, - const char *var_name) +struct gdim_list_entry *lookup_gdim_list(struct list_head *gdim_list, + const char *var_name) { - if (!gdim_list) return NULL; + if(!gdim_list) + return NULL; struct gdim_list_entry *e; list_for_each_entry(e, gdim_list, struct gdim_list_entry, entry) { - if (0==strcmp(e->var_name, var_name)) return e; + if(0 == strcmp(e->var_name, var_name)) + return e; } return NULL; } void update_gdim_list(struct list_head *gdim_list, const char *var_name, - int ndim, uint64_t *gdim) + int ndim, uint64_t *gdim) { struct gdim_list_entry *e = lookup_gdim_list(gdim_list, var_name); - if (!e) { + if(!e) { // add new entry - e = (struct gdim_list_entry*)malloc(sizeof(*e)); - e->var_name = malloc(strlen(var_name)+1); + e = (struct gdim_list_entry *)malloc(sizeof(*e)); + e->var_name = malloc(strlen(var_name) + 1); strcpy(e->var_name, var_name); - list_add(&e->entry, gdim_list); + list_add(&e->entry, gdim_list); } // update entry @@ -1565,41 +1623,40 @@ void update_gdim_list(struct list_head *gdim_list, const char *var_name, } void set_global_dimension(struct list_head *gdim_list, const char *var_name, - const struct global_dimension *default_gdim, struct global_dimension *gdim) + const struct global_dimension *default_gdim, + struct global_dimension *gdim) { struct gdim_list_entry *e = lookup_gdim_list(gdim_list, var_name); - if (e) { + if(e) { memcpy(gdim, &e->gdim, sizeof(struct global_dimension)); } else { memcpy(gdim, default_gdim, sizeof(struct global_dimension)); } } -int global_dimension_equal(const struct global_dimension* gdim1, - const struct global_dimension* gdim2) +int global_dimension_equal(const struct global_dimension *gdim1, + const struct global_dimension *gdim2) { int i; - for (i = 0; i < gdim1->ndim; i++) { - if (gdim1->sizes.c[i] != gdim2->sizes.c[i]) + for(i = 0; i < gdim1->ndim; i++) { + if(gdim1->sizes.c[i] != gdim2->sizes.c[i]) return 0; } return 1; } -char ** addr_str_buf_to_list( - char * buf, int num_addrs) +char **addr_str_buf_to_list(char *buf, int num_addrs) { int i; char **ret = malloc(num_addrs * sizeof(*ret)); - if (ret == NULL) return NULL; + if(ret == NULL) + return NULL; ret[0] = (char *)buf; - for (i = 1; i < num_addrs; i++) - { - char * a = ret[i-1]; + for(i = 1; i < num_addrs; i++) { + char *a = ret[i - 1]; ret[i] = a + strlen(a) + 1; } return ret; } - diff --git a/src/util.c b/src/util.c index 168f92c2..8f29cb86 100644 --- a/src/util.c +++ b/src/util.c @@ -1,35 +1,40 @@ /* - * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers University - * All rights reserved. + * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers + * University All rights reserved. * - * Redistribution and use in source and binary forms, with or without modification, are permitted provided - * that the following conditions are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * - Redistributions of source code must retain the above copyright notice, this list of conditions and - * the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and - * the following disclaimer in the documentation and/or other materials provided with the distribution. - * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers University, nor the names of its - * contributors may be used to endorse or promote products derived from this software without specific prior - * written permission. + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers + * University, nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written + * permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* -* Ciprian Docan (2009) TASSL Rutgers University -* docan@cac.rutgers.edu -* Tong Jin (2011) TASSL Rutgers University -* tjin@cac.rutgers.edu -*/ + * Ciprian Docan (2009) TASSL Rutgers University + * docan@cac.rutgers.edu + * Tong Jin (2011) TASSL Rutgers University + * tjin@cac.rutgers.edu + */ #include #include @@ -40,79 +45,76 @@ size_t str_len(const char *str) { - if(str) - return strlen(str); - else - return 0; + if(str) + return strlen(str); + else + return 0; } char *str_append_const(char *str, const char *msg) { - int len, fix_str; + int len, fix_str; - len = str_len(str) + str_len(msg) + 1; - fix_str = (str == 0); - str = realloc(str, len); - if(fix_str) - *str = '\0'; - if(str) - strcat(str, msg); + len = str_len(str) + str_len(msg) + 1; + fix_str = (str == 0); + str = realloc(str, len); + if(fix_str) + *str = '\0'; + if(str) + strcat(str, msg); - return str; + return str; } char *str_append(char *str, char *msg) { - str = str_append_const(str, msg); + str = str_append_const(str, msg); - free(msg); - return str; + free(msg); + return str; } /* - * Our own implementation of the asprintf functionality + * Our own implementation of the asprintf functionality */ char *alloc_sprintf(const char *fmt_str, ...) { va_list va_args_tmp, va_args; int size; char *str; - + va_start(va_args_tmp, fmt_str); va_copy(va_args, va_args_tmp); size = vsnprintf(NULL, 0, fmt_str, va_args_tmp); va_end(va_args_tmp); str = malloc(sizeof(*str) * (size + 1)); - vsprintf(str, fmt_str, va_args); + vsprintf(str, fmt_str, va_args); va_end(va_args); - return(str); - + return (str); } /******************************************************* Processing parameter lists **********************************************************/ -static char * remove_whitespace (char *start, char *end) +static char *remove_whitespace(char *start, char *end) { char *s = start; char *e = end; - //int orig_len = (int) (e-s); + // int orig_len = (int) (e-s); int final_len; char *res; // remove front whitespace (but do not go far beyond the end) - while (s <= e && - (*s==' ' || *s=='\t' || *s=='\n') - ) s++; - if (s <= e) { // there is some text + while(s <= e && (*s == ' ' || *s == '\t' || *s == '\n')) + s++; + if(s <= e) { // there is some text // remove tail whitespace - while (s <= e && - (*e==' ' || *e=='\t' || *e=='\n') - ) e--; - // create result - final_len = e - s + 1; // length of result - if (final_len > 0) { - res = (char *) malloc (final_len + 1); // allocate space s..e and \0 + while(s <= e && (*e == ' ' || *e == '\t' || *e == '\n')) + e--; + // create result + final_len = e - s + 1; // length of result + if(final_len > 0) { + res = (char *)malloc(final_len + 1); // allocate space s..e and \0 memcpy(res, s, final_len); res[final_len] = 0; } else { @@ -126,101 +128,101 @@ static char * remove_whitespace (char *start, char *end) return res; } - /* Split a line at = sign into name and value pair Remove " ", TAB and Newline from around names and values Return NULL for name and value if there is no = sign in line Return newly allocated strings otherwise Used by: esimmon_internal_text_to_name_value_pairs */ -static void splitnamevalue (const char * line, int linelen, char **name, char **value) +static void splitnamevalue(const char *line, int linelen, char **name, + char **value) { char *equal; // position of first = sign in line - equal = strchr (line, '='); - if (equal && equal != line) { + equal = strchr(line, '='); + if(equal && equal != line) { /* 1. name */ // from first char to before = - *name = remove_whitespace ((char*)line, equal-1); + *name = remove_whitespace((char *)line, equal - 1); /* 2. value */ // from after = to the last character of line - *value = remove_whitespace (equal+1, (char*)line+linelen-1); + *value = remove_whitespace(equal + 1, (char *)line + linelen - 1); - } else if (equal != line) { + } else if(equal != line) { /* check if it as name without = value statement */ - *name = remove_whitespace ((char*)line, (char*)line+linelen-1); + *name = remove_whitespace((char *)line, (char *)line + linelen - 1); *value = NULL; - } else { - // funny text starting with =. E.g. "=value" + } else { + // funny text starting with =. E.g. "=value" *name = NULL; *value = NULL; } } -struct name_value_pair* text_to_nv_pairs (const char * text) +struct name_value_pair *text_to_nv_pairs(const char *text) { /* Process a multi-line and/or ;-separated text and create a list - of name=value pairs from each line which has a - name = value - pattern. Whitespaces are removed. + of name=value pairs from each line which has a + name = value + pattern. Whitespaces are removed. "X = 1 - Y = 2" + Y = 2" is not valid because of missing ';', but "X=1; Y=5; - Z=apple" + Z=apple" is valid */ - char *name, *value; + char *name, *value; char *item, *delim; int len; char line[256]; struct name_value_pair *res = NULL, *last = NULL, *pair; - if (!text) return res; + if(!text) + return res; - item = (char *)text; - while (item) { - delim = strchr (item, ';'); - if (delim) - len = (int) (delim-item); - else - len = strlen (item); + item = (char *)text; + while(item) { + delim = strchr(item, ';'); + if(delim) + len = (int)(delim - item); + else + len = strlen(item); - strncpy (line, item, len); + strncpy(line, item, len); line[len] = '\0'; splitnamevalue(line, len, &name, &value); - if (name) { - pair = (struct name_value_pair*) malloc (sizeof(struct name_value_pair)); + if(name) { + pair = (struct name_value_pair *)malloc( + sizeof(struct name_value_pair)); pair->name = name; pair->value = value; pair->next = NULL; - if (last) { + if(last) { last->next = pair; last = pair; } else { - res = pair; + res = pair; last = pair; } } - if (delim && delim+1 != 0) - item = delim+1; + if(delim && delim + 1 != 0) + item = delim + 1; else item = NULL; } return res; } - -void free_nv_pairs (struct name_value_pair* pairs) +void free_nv_pairs(struct name_value_pair *pairs) { struct name_value_pair *p; - while (pairs) { + while(pairs) { free(pairs->name); free(pairs->value); p = pairs; - pairs=pairs->next; + pairs = pairs->next; free(p); } } - diff --git a/tests/server.c b/tests/server.c index 3ca3ca72..e09e7805 100644 --- a/tests/server.c +++ b/tests/server.c @@ -1,25 +1,24 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include +#include #include #include #include -#include -#include - -int main(int argc, char** argv) +int main(int argc, char **argv) { if(argc != 2) { fprintf(stderr, "Usage: %s \n", argv[0]); return -1; } - - char* listen_addr_str = argv[1]; + char *listen_addr_str = argv[1]; dspaces_provider_t s = dspaces_PROVIDER_NULL; @@ -32,16 +31,16 @@ int main(int argc, char** argv) MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); int ret = dspaces_server_init(listen_addr_str, gcomm, &s); - if(ret != 0) return ret; + if(ret != 0) + return ret; // make margo wait for finalize dspaces_server_fini(s); if(rank == 0) { fprintf(stderr, "Server is all done!\n"); - } - + } + MPI_Finalize(); return 0; - } diff --git a/tests/terminator.c b/tests/terminator.c index 7d15464f..72d59606 100644 --- a/tests/terminator.c +++ b/tests/terminator.c @@ -1,16 +1,17 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "mpi.h" +#include "timer.h" +#include +#include #include #include #include -#include -#include -#include "timer.h" -#include "mpi.h" int main(int argc, char **argv) { @@ -25,10 +26,9 @@ int main(int argc, char **argv) dspaces_kill(ds); - dspaces_fini(ds); + dspaces_fini(ds); MPI_Finalize(); - return(0); - + return (0); } diff --git a/tests/test_get_run.c b/tests/test_get_run.c index 52637a66..25354d4f 100644 --- a/tests/test_get_run.c +++ b/tests/test_get_run.c @@ -1,21 +1,22 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "mpi.h" +#include "timer.h" +#include #include +#include #include #include #include -#include -#include -#include "timer.h" -#include "mpi.h" //# of processors in fortran direction static int np[10] = {0}; -//block size per processor per direction +// block size per processor per direction static uint64_t sp[10] = {0}; //# of interations static int timesteps_; @@ -34,126 +35,126 @@ static size_t elem_size_; static char transport_type_str_[256]; -static double* allocate_nd(int dims) +static double *allocate_nd(int dims) { - double* tmp = NULL; - int i = 0; - uint64_t size = 1; - for(i = 0; i < dims; i++){ - size *= sp[i]; - } - tmp = (double*)malloc(elem_size_ * size); - return tmp; + double *tmp = NULL; + int i = 0; + uint64_t size = 1; + for(i = 0; i < dims; i++) { + size *= sp[i]; + } + tmp = (double *)malloc(elem_size_ * size); + return tmp; } static void set_offset_nd(int rank, int dims) { - int i = 0, j = 0; - for(i = 0; i < dims; i++){ - int tmp = rank; - for(j = 0; j < i; j++) - tmp /= np[j]; - off[i] = tmp % np[i] * sp[i]; - } + int i = 0, j = 0; + for(i = 0; i < dims; i++) { + int tmp = rank; + for(j = 0; j < i; j++) + tmp /= np[j]; + off[i] = tmp % np[i] * sp[i]; + } } -int check_data(const char *var_name, double *buf, int num_elem, int rank, int ts) +int check_data(const char *var_name, double *buf, int num_elem, int rank, + int ts) { - double max, min, sum, avg; - int i; - int cnt = 0; + double max, min, sum, avg; + int i; + int cnt = 0; - if (num_elem <= 0) { - return -EINVAL; - } - max = min = sum = buf[0]; - for (i = 0; i < num_elem; i++) { - if (max < buf[i]) - max = buf[i]; - if (min > buf [i]) - min = buf[i]; - sum += buf[i]; - if (buf[i] != ts) { - cnt++; - } + if(num_elem <= 0) { + return -EINVAL; + } + max = min = sum = buf[0]; + for(i = 0; i < num_elem; i++) { + if(max < buf[i]) + max = buf[i]; + if(min > buf[i]) + min = buf[i]; + sum += buf[i]; + if(buf[i] != ts) { + cnt++; } - avg = sum / num_elem; - if (cnt > 0) { - fprintf(stderr, "%s(): var= %s, rank= %d, ts= %d, " + } + avg = sum / num_elem; + if(cnt > 0) { + fprintf(stderr, + "%s(): var= %s, rank= %d, ts= %d, " "error elem cnt= %d, total elem= %d\n", - __func__, var_name, rank, ts, cnt, num_elem); - } + __func__, var_name, rank, ts, cnt, num_elem); + } - return cnt; + return cnt; } -static int couple_read_nd(dspaces_client_t client, unsigned int ts, int num_vars, int dims) +static int couple_read_nd(dspaces_client_t client, unsigned int ts, + int num_vars, int dims) { - double **data_tab = (double **)malloc(sizeof(double*) * num_vars); - char var_name[128]; - int ret = 0; + double **data_tab = (double **)malloc(sizeof(double *) * num_vars); + char var_name[128]; + int ret = 0; int err = 0; int i; - for(i = 0; i < num_vars; i++){ - data_tab[i] = NULL; - } - - set_offset_nd(rank_, dims); - uint64_t dims_size = 1; - int elem_size = elem_size_; - uint64_t lb[10] = {0}, ub[10] = {0}; - for(i = 0; i < dims; i++){ - lb[i] = off[i]; - ub[i] = off[i] + sp[i] - 1; - dims_size *= sp[i]; - } - double tm_st, tm_end, tm_max, tm_diff; - int root = 0; - - - //allocate data - double *data = NULL; - for(i = 0; i < num_vars; i++){ - data = allocate_nd(dims); - if(data == NULL){ - fprintf(stderr, "%s(): allocate_2d() failed.\n", __func__); - return -1; // TODO: free buffers - } - memset(data, 0, elem_size_ * dims_size); - data_tab[i] = data; - } - - MPI_Barrier(gcomm_); + for(i = 0; i < num_vars; i++) { + data_tab[i] = NULL; + } + + set_offset_nd(rank_, dims); + uint64_t dims_size = 1; + int elem_size = elem_size_; + uint64_t lb[10] = {0}, ub[10] = {0}; + for(i = 0; i < dims; i++) { + lb[i] = off[i]; + ub[i] = off[i] + sp[i] - 1; + dims_size *= sp[i]; + } + double tm_st, tm_end, tm_max, tm_diff; + int root = 0; + + // allocate data + double *data = NULL; + for(i = 0; i < num_vars; i++) { + data = allocate_nd(dims); + if(data == NULL) { + fprintf(stderr, "%s(): allocate_2d() failed.\n", __func__); + return -1; // TODO: free buffers + } + memset(data, 0, elem_size_ * dims_size); + data_tab[i] = data; + } + + MPI_Barrier(gcomm_); tm_st = timer_read(&timer_); - for(i = 0; i < num_vars; i++){ - sprintf(var_name, "mnd_%d", i); - err = dspaces_get(client, var_name, ts, elem_size, dims, lb, ub, - data_tab[i], -1); - if(err != 0){ - fprintf(stderr, "dspaces_get() returned error %d\n", err); - return err; - } - - } - tm_end = timer_read(&timer_); - - tm_diff = tm_end-tm_st; - MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); - - if (rank_ == root) { - fprintf(stdout, "TS= %u read MAX time= %lf\n", - ts, tm_max); - } - - for (i = 0; i < num_vars; i++) { - sprintf(var_name, "mnd_%d", i); - err = check_data(var_name, data_tab[i],dims_size*elem_size_/sizeof(double), - rank_, ts); + for(i = 0; i < num_vars; i++) { + sprintf(var_name, "mnd_%d", i); + err = dspaces_get(client, var_name, ts, elem_size, dims, lb, ub, + data_tab[i], -1); + if(err != 0) { + fprintf(stderr, "dspaces_get() returned error %d\n", err); + return err; + } + } + tm_end = timer_read(&timer_); + + tm_diff = tm_end - tm_st; + MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); + + if(rank_ == root) { + fprintf(stdout, "TS= %u read MAX time= %lf\n", ts, tm_max); + } + + for(i = 0; i < num_vars; i++) { + sprintf(var_name, "mnd_%d", i); + err = check_data(var_name, data_tab[i], + dims_size * elem_size_ / sizeof(double), rank_, ts); if(err > 0) { - ret = -EIO; + ret = -EIO; } - if (data_tab[i]) { + if(data_tab[i]) { free(data_tab[i]); } } @@ -162,32 +163,31 @@ static int couple_read_nd(dspaces_client_t client, unsigned int ts, int num_vars return ret; } -int test_get_run(int ndims, int* npdim, - uint64_t *spdim, int timestep, size_t elem_size, int num_vars, int terminate, - MPI_Comm gcomm) +int test_get_run(int ndims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int terminate, MPI_Comm gcomm) { - gcomm_ = gcomm; - elem_size_ = elem_size; - timesteps_ = timestep; + gcomm_ = gcomm; + elem_size_ = elem_size; + timesteps_ = timestep; - dspaces_client_t ndcl = dspaces_CLIENT_NULL; + dspaces_client_t ndcl = dspaces_CLIENT_NULL; - hg_return_t hret = HG_SUCCESS; + hg_return_t hret = HG_SUCCESS; int err, ret = 0; - int i; - for(i = 0; i < ndims; i++){ + int i; + for(i = 0; i < ndims; i++) { np[i] = npdim[i]; sp[i] = spdim[i]; - } + } - timer_init(&timer_, 1); + timer_init(&timer_, 1); timer_start(&timer_); - double tm_st, tm_end; - tm_st = timer_read(&timer_); - - MPI_Comm_rank(gcomm_, &rank_); + double tm_st, tm_end; + tm_st = timer_read(&timer_); + + MPI_Comm_rank(gcomm_, &rank_); MPI_Comm_size(gcomm_, &nproc_); ret = dspaces_init(rank_, &ndcl); @@ -196,24 +196,24 @@ int test_get_run(int ndims, int* npdim, goto error; } - tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); - - unsigned int ts; - for(ts = 1; ts <= timesteps_; ts++){ - err = couple_read_nd(ndcl, ts, num_vars, ndims); - if(err != 0) { - ret = -1; - } + tm_end = timer_read(&timer_); + fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); - } - - MPI_Barrier(gcomm_); + unsigned int ts; + for(ts = 1; ts <= timesteps_; ts++) { + err = couple_read_nd(ndcl, ts, num_vars, ndims); + if(err != 0) { + ret = -1; + } + } - if(rank_ == 0) { - fprintf(stdout, "%s(): done\n", __func__); - } - tm_st = timer_read(&timer_); + MPI_Barrier(gcomm_); + + if(rank_ == 0) { + fprintf(stdout, "%s(): done\n", __func__); + } + tm_st = timer_read(&timer_); if(rank_ == 0 && terminate) { fprintf(stderr, "Reader sending kill signal to server.\n"); @@ -221,15 +221,15 @@ int test_get_run(int ndims, int* npdim, } dspaces_fini(ndcl); - tm_end = timer_read(&timer_); + tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); + fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); return ret; - error: +error: dspaces_fini(ndcl); - + return ret; - } diff --git a/tests/test_put_run.c b/tests/test_put_run.c index b30276cd..7c887616 100644 --- a/tests/test_put_run.c +++ b/tests/test_put_run.c @@ -1,20 +1,21 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "mpi.h" +#include "timer.h" +#include +#include #include #include #include -#include -#include -#include "timer.h" -#include "mpi.h" //# of processors in fortran direction static int np[10] = {0}; -//block size per processor per direction +// block size per processor per direction static uint64_t sp[10] = {0}; //# of interations static int timesteps_; @@ -31,104 +32,103 @@ static MPI_Comm gcomm_; static size_t elem_size_; -static double* allocate_nd(int dims) +static double *allocate_nd(int dims) { - double* tmp = NULL; - int i = 0; - uint64_t size = 1; - for(i = 0; i < dims; i++){ - size *= sp[i]; - } - tmp = (double*)malloc(elem_size_ * size); - return tmp; + double *tmp = NULL; + int i = 0; + uint64_t size = 1; + for(i = 0; i < dims; i++) { + size *= sp[i]; + } + tmp = (double *)malloc(elem_size_ * size); + return tmp; } static void set_offset_nd(int rank, int dims) { - int i = 0, j = 0; - for(i = 0; i < dims; i++){ - int tmp = rank; - for(j = 0; j < i; j++) - tmp /= np[j]; - off[i] = tmp % np[i] * sp[i]; - } + int i = 0, j = 0; + for(i = 0; i < dims; i++) { + int tmp = rank; + for(j = 0; j < i; j++) + tmp /= np[j]; + off[i] = tmp % np[i] * sp[i]; + } } - static int generate_nd(double *mnd, unsigned int ts, int dims) { - //double value = 1.0*(rank_) + 0.0001*ts; - double value = ts; + // double value = 1.0*(rank_) + 0.0001*ts; + double value = ts; int i; - uint64_t mnd_size = 1; + uint64_t mnd_size = 1; for(i = 0; i < dims; i++) mnd_size *= sp[i]; - mnd_size = mnd_size * elem_size_ / sizeof(double); + mnd_size = mnd_size * elem_size_ / sizeof(double); for(i = 0; i < mnd_size; i++) - *(mnd+i) = value; + *(mnd + i) = value; return 0; } -static int couple_write_nd(dspaces_client_t ndph, unsigned int ts, int num_vars, int dims, int local_mode) +static int couple_write_nd(dspaces_client_t ndph, unsigned int ts, int num_vars, + int dims, int local_mode) { - double **data_tab = (double **)malloc(sizeof(double *) * num_vars); - char var_name[128]; - int i; - int err; - for(i = 0; i < num_vars; i++) - data_tab[i] = NULL; - - set_offset_nd(rank_, dims); - int elem_size = elem_size_; - uint64_t lb[10] = {0}, ub[10] = {0}; - for(i = 0; i < dims; i++){ - lb[i] = off[i]; - ub[i] = off[i] + sp[i] - 1; - } - double tm_st, tm_end, tm_max, tm_diff; - int root = 0; - - //allocate data - double *data = NULL; - for(i = 0; i < num_vars; i++){ - data = allocate_nd(dims); - if(data == NULL){ - fprintf(stderr, "%s(): allocate_nd() failed.\n", __func__); + double **data_tab = (double **)malloc(sizeof(double *) * num_vars); + char var_name[128]; + int i; + int err; + for(i = 0; i < num_vars; i++) + data_tab[i] = NULL; + + set_offset_nd(rank_, dims); + int elem_size = elem_size_; + uint64_t lb[10] = {0}, ub[10] = {0}; + for(i = 0; i < dims; i++) { + lb[i] = off[i]; + ub[i] = off[i] + sp[i] - 1; + } + double tm_st, tm_end, tm_max, tm_diff; + int root = 0; + + // allocate data + double *data = NULL; + for(i = 0; i < num_vars; i++) { + data = allocate_nd(dims); + if(data == NULL) { + fprintf(stderr, "%s(): allocate_nd() failed.\n", __func__); return -1; // TODO: free buffers - } - - generate_nd(data, ts, dims); - data_tab[i] = data; - } + } - MPI_Barrier(gcomm_); + generate_nd(data, ts, dims); + data_tab[i] = data; + } + + MPI_Barrier(gcomm_); tm_st = timer_read(&timer_); - for(i = 0; i < num_vars; i++){ - sprintf(var_name, "mnd_%d", i); - if(!local_mode) - err = dspaces_put(ndph, var_name, ts, elem_size, dims, lb, ub, data_tab[i]); - else - err = dspaces_put_local(ndph, var_name, ts, elem_size, dims, lb, ub, data_tab[i]); - if(err!=0){ - fprintf(stderr, "dspaces_put returned error %d", err); - return err; - } - - } - tm_end = timer_read(&timer_); - - - tm_diff = tm_end-tm_st; - MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); - - if (rank_ == root) { - fprintf(stdout, "TS= %u write MAX time= %lf\n", - ts, tm_max); + for(i = 0; i < num_vars; i++) { + sprintf(var_name, "mnd_%d", i); + if(!local_mode) + err = dspaces_put(ndph, var_name, ts, elem_size, dims, lb, ub, + data_tab[i]); + else + err = dspaces_put_local(ndph, var_name, ts, elem_size, dims, lb, ub, + data_tab[i]); + if(err != 0) { + fprintf(stderr, "dspaces_put returned error %d", err); + return err; + } + } + tm_end = timer_read(&timer_); + + tm_diff = tm_end - tm_st; + MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); + + if(rank_ == root) { + fprintf(stdout, "TS= %u write MAX time= %lf\n", ts, tm_max); } - for (i = 0; i < num_vars; i++) { - if (data_tab[i]) { + for(i = 0; i < num_vars; i++) { + if(data_tab[i]) { free(data_tab[i]); } } @@ -137,55 +137,54 @@ static int couple_write_nd(dspaces_client_t ndph, unsigned int ts, int num_vars, return 0; } -int test_put_run(int ndims, int* npdim, - uint64_t *spdim, int timestep, size_t elem_size, int num_vars, int local_mode, - int terminate, MPI_Comm gcomm) +int test_put_run(int ndims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int local_mode, int terminate, + MPI_Comm gcomm) { - gcomm_ = gcomm; - elem_size_ = elem_size; - timesteps_ = timestep; + gcomm_ = gcomm; + elem_size_ = elem_size; + timesteps_ = timestep; int i, ret; - dspaces_client_t ndcl = dspaces_CLIENT_NULL; + dspaces_client_t ndcl = dspaces_CLIENT_NULL; - ret = 0; + ret = 0; - for(i = 0; i < ndims; i++){ + for(i = 0; i < ndims; i++) { np[i] = npdim[i]; sp[i] = spdim[i]; - } + } - timer_init(&timer_, 1); + timer_init(&timer_, 1); timer_start(&timer_); - double tm_st, tm_end; - tm_st = timer_read(&timer_); + double tm_st, tm_end; + tm_st = timer_read(&timer_); + + MPI_Comm_rank(gcomm_, &rank_); - MPI_Comm_rank(gcomm_, &rank_); - ret = dspaces_init(rank_, &ndcl); - tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); + tm_end = timer_read(&timer_); + fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); MPI_Comm_size(gcomm_, &nproc_); + unsigned int ts; + for(ts = 1; ts <= timesteps_; ts++) { + ret = couple_write_nd(ndcl, ts, num_vars, ndims, local_mode); + if(ret != 0) { + ret = -1; + goto error; + } + } - unsigned int ts; - for(ts = 1; ts <= timesteps_; ts++){ - ret = couple_write_nd(ndcl, ts, num_vars, ndims, local_mode); - if(ret!=0){ - ret = -1; - goto error; - } - - } - - MPI_Barrier(gcomm_); - if(rank_ == 0) { - fprintf(stdout, "%s(): done\n", __func__); - } - tm_st = timer_read(&timer_); + MPI_Barrier(gcomm_); + if(rank_ == 0) { + fprintf(stdout, "%s(): done\n", __func__); + } + tm_st = timer_read(&timer_); if(rank_ == 0 && terminate) { fprintf(stderr, "Writer sending kill signal to server.\n"); @@ -194,17 +193,16 @@ int test_put_run(int ndims, int* npdim, dspaces_fini(ndcl); - tm_end = timer_read(&timer_); + tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); - + fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); return 0; - error: +error: dspaces_fini(ndcl); return ret; - } diff --git a/tests/test_reader.c b/tests/test_reader.c index b89bfe72..ca7fd29f 100644 --- a/tests/test_reader.c +++ b/tests/test_reader.c @@ -1,35 +1,43 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ - -#include +#include "mpi.h" #include +#include #include #include -#include "mpi.h" -extern int test_get_run(int dims, - int* npdim, uint64_t *spdim, int timestep, - size_t elem_size, int num_vars, int terminate, MPI_Comm gcomm); +extern int test_get_run(int dims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int terminate, + MPI_Comm gcomm); void print_usage() { - fprintf(stderr, "Usage: test_reader np[0] .. np[dims-1] sp[0] ... sp[dims-1] [-s ] [-c ] [-t]\n" -" dims - number of data dimensions. Must be at least one\n" -" np[i] - the number of processes in the ith dimension. The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" -" sp[i] - the per-process data size in the ith dimension\n" -" timesteps - the number of timestep iterations written\n" -" -s - the number of bytes in each element. Defaults to 8\n" -" -c - the number of variables written in each iteration. Defaults to one\n" -" -t - send server termination signal after reading is complete\n"); + fprintf( + stderr, + "Usage: test_reader np[0] .. np[dims-1] sp[0] ... sp[dims-1] " + " [-s ] [-c ] [-t]\n" + " dims - number of data dimensions. Must be at least " + "one\n" + " np[i] - the number of processes in the ith dimension. " + "The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" + " sp[i] - the per-process data size in the ith " + "dimension\n" + " timesteps - the number of timestep iterations written\n" + " -s - the number of bytes in each element. Defaults " + "to 8\n" + " -c - the number of variables written in each " + "iteration. Defaults to one\n" + " -t - send server termination signal after reading " + "is complete\n"); } -int parse_args(int argc, char **argv, - int *dims, int *npdim, uint64_t *spdim, int *timestep, - size_t *elem_size, int *num_vars, int *terminate) +int parse_args(int argc, char **argv, int *dims, int *npdim, uint64_t *spdim, + int *timestep, size_t *elem_size, int *num_vars, int *terminate) { char **argp; int i; @@ -41,15 +49,15 @@ int parse_args(int argc, char **argv, if(argc > 1) { *dims = atoi(argv[1]); } - + if(argc < 3 + (*dims * 2)) { fprintf(stderr, "Not enough arguments.\n"); print_usage(); - return(-1); + return (-1); } argp = &argv[2]; - + for(i = 0; i < *dims; i++) { npdim[i] = atoi(*argp); argp++; @@ -58,7 +66,7 @@ int parse_args(int argc, char **argv, spdim[i] = strtoull(*argp, NULL, 10); argp++; } - + *timestep = atoi(*argp); argp++; @@ -67,17 +75,17 @@ int parse_args(int argc, char **argv, if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *elem_size = atoi(*(argp+1)); + *elem_size = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-c") == 0) { if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *num_vars = atoi(*(argp+1)); + *num_vars = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-t") == 0) { *terminate = 1; @@ -85,7 +93,7 @@ int parse_args(int argc, char **argv, } else { fprintf(stderr, "Unknown argument: %s\n", *argp); print_usage(); - return(-1); + return (-1); } } @@ -93,58 +101,61 @@ int parse_args(int argc, char **argv, fprintf(stderr, "Warning: ignoring extraneous argument '%s'.\n", *argp); } - return(0); + return (0); } int main(int argc, char **argv) { - int err; - int nprocs, rank; - MPI_Comm gcomm; + int err; + int nprocs, rank; + MPI_Comm gcomm; int i, ret; - int npapp; // number of application processes - int np[10] = {0}; //number of processes in each dimension - uint64_t sp[10] = {0}; //block size per process in each dimension - int timestep; // number of iterations - int dims; // number of dimensions - size_t elem_size; // Optional: size of one element in the global array. Default value is 8 (bytes). - int num_vars; // Optional: number of variables to be shared in the testing. Default value is 1. + int npapp; // number of application processes + int np[10] = {0}; // number of processes in each dimension + uint64_t sp[10] = {0}; // block size per process in each dimension + int timestep; // number of iterations + int dims; // number of dimensions + size_t elem_size; // Optional: size of one element in the global array. + // Default value is 8 (bytes). + int num_vars; // Optional: number of variables to be shared in the testing. + // Default value is 1. int terminate; // Optional: send terminate signal to server after read - if (parse_args(argc, argv, &dims, np, sp, - ×tep, &elem_size, &num_vars, &terminate) != 0) { - goto err_out; - } + if(parse_args(argc, argv, &dims, np, sp, ×tep, &elem_size, &num_vars, + &terminate) != 0) { + goto err_out; + } npapp = 1; for(i = 0; i < dims; i++) { npapp *= np[i]; } - // Using SPMD style programming - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Barrier(MPI_COMM_WORLD); - gcomm = MPI_COMM_WORLD; + // Using SPMD style programming + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Barrier(MPI_COMM_WORLD); + gcomm = MPI_COMM_WORLD; - int color = 1; - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); + int color = 1; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); if(npapp != nprocs) { - fprintf(stderr, "Product of np[i] args must equal number of MPI processes!\n"); + fprintf(stderr, + "Product of np[i] args must equal number of MPI processes!\n"); print_usage(); - return(-1); + return (-1); } - // Run as data reader + // Run as data reader - ret = test_get_run(dims, np, - sp, timestep, elem_size, num_vars, terminate, gcomm); + ret = test_get_run(dims, np, sp, timestep, elem_size, num_vars, terminate, + gcomm); - MPI_Barrier(gcomm); - MPI_Finalize(); + MPI_Barrier(gcomm); + MPI_Finalize(); if(rank == 0) { fprintf(stderr, "That's all from test_reader, folks!\n"); @@ -154,8 +165,8 @@ int main(int argc, char **argv) goto err_out; } - return ret; + return ret; err_out: - fprintf(stderr, "test_reader rank %d has failed!\n", rank); - return -1; + fprintf(stderr, "test_reader rank %d has failed!\n", rank); + return -1; } diff --git a/tests/test_sub.c b/tests/test_sub.c index 1ffcc1e1..f2295348 100644 --- a/tests/test_sub.c +++ b/tests/test_sub.c @@ -1,35 +1,43 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ - -#include +#include "mpi.h" #include +#include #include #include -#include "mpi.h" -extern int test_sub_run(int dims, - int* npdim, uint64_t *spdim, int timestep, - size_t elem_size, int num_vars, int terminate, MPI_Comm gcomm); +extern int test_sub_run(int dims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int terminate, + MPI_Comm gcomm); void print_usage() { - fprintf(stderr, "Usage: test_sb np[0] .. np[dims-1] sp[0] ... sp[dims-1] [-s ] [-c ] [-t]\n" -" dims - number of data dimensions. Must be at least one\n" -" np[i] - the number of processes in the ith dimension. The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" -" sp[i] - the per-process data size in the ith dimension\n" -" timesteps - the number of timestep iterations written\n" -" -s - the number of bytes in each element. Defaults to 8\n" -" -c - the number of variables written in each iteration. Defaults to one\n" -" -t - send server termination signal after reading is complete\n"); + fprintf( + stderr, + "Usage: test_sb np[0] .. np[dims-1] sp[0] ... sp[dims-1] " + " [-s ] [-c ] [-t]\n" + " dims - number of data dimensions. Must be at least " + "one\n" + " np[i] - the number of processes in the ith dimension. " + "The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" + " sp[i] - the per-process data size in the ith " + "dimension\n" + " timesteps - the number of timestep iterations written\n" + " -s - the number of bytes in each element. Defaults " + "to 8\n" + " -c - the number of variables written in each " + "iteration. Defaults to one\n" + " -t - send server termination signal after reading " + "is complete\n"); } -int parse_args(int argc, char **argv, - int *dims, int *npdim, uint64_t *spdim, int *timestep, - size_t *elem_size, int *num_vars, int *terminate) +int parse_args(int argc, char **argv, int *dims, int *npdim, uint64_t *spdim, + int *timestep, size_t *elem_size, int *num_vars, int *terminate) { char **argp; int i; @@ -41,15 +49,15 @@ int parse_args(int argc, char **argv, if(argc > 1) { *dims = atoi(argv[1]); } - + if(argc < 3 + (*dims * 2)) { fprintf(stderr, "Not enough arguments.\n"); print_usage(); - return(-1); + return (-1); } argp = &argv[2]; - + for(i = 0; i < *dims; i++) { npdim[i] = atoi(*argp); argp++; @@ -58,7 +66,7 @@ int parse_args(int argc, char **argv, spdim[i] = strtoull(*argp, NULL, 10); argp++; } - + *timestep = atoi(*argp); argp++; @@ -67,17 +75,17 @@ int parse_args(int argc, char **argv, if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *elem_size = atoi(*(argp+1)); + *elem_size = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-c") == 0) { if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *num_vars = atoi(*(argp+1)); + *num_vars = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-t") == 0) { *terminate = 1; @@ -85,7 +93,7 @@ int parse_args(int argc, char **argv, } else { fprintf(stderr, "Unknown argument: %s\n", *argp); print_usage(); - return(-1); + return (-1); } } @@ -93,58 +101,61 @@ int parse_args(int argc, char **argv, fprintf(stderr, "Warning: ignoring extraneous argument '%s'.\n", *argp); } - return(0); + return (0); } int main(int argc, char **argv) { - int err; - int nprocs, rank; - MPI_Comm gcomm; + int err; + int nprocs, rank; + MPI_Comm gcomm; int i, ret; - int npapp; // number of application processes - int np[10] = {0}; //number of processes in each dimension - uint64_t sp[10] = {0}; //block size per process in each dimension - int timestep; // number of iterations - int dims; // number of dimensions - size_t elem_size; // Optional: size of one element in the global array. Default value is 8 (bytes). - int num_vars; // Optional: number of variables to be shared in the testing. Default value is 1. + int npapp; // number of application processes + int np[10] = {0}; // number of processes in each dimension + uint64_t sp[10] = {0}; // block size per process in each dimension + int timestep; // number of iterations + int dims; // number of dimensions + size_t elem_size; // Optional: size of one element in the global array. + // Default value is 8 (bytes). + int num_vars; // Optional: number of variables to be shared in the testing. + // Default value is 1. int terminate; // Optional: send terminate signal to server after read - if (parse_args(argc, argv, &dims, np, sp, - ×tep, &elem_size, &num_vars, &terminate) != 0) { - goto err_out; - } + if(parse_args(argc, argv, &dims, np, sp, ×tep, &elem_size, &num_vars, + &terminate) != 0) { + goto err_out; + } npapp = 1; for(i = 0; i < dims; i++) { npapp *= np[i]; } - // Using SPMD style programming - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Barrier(MPI_COMM_WORLD); - gcomm = MPI_COMM_WORLD; + // Using SPMD style programming + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Barrier(MPI_COMM_WORLD); + gcomm = MPI_COMM_WORLD; - int color = 1; - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); + int color = 1; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); if(npapp != nprocs) { - fprintf(stderr, "Product of np[i] args must equal number of MPI processes!\n"); + fprintf(stderr, + "Product of np[i] args must equal number of MPI processes!\n"); print_usage(); - return(-1); + return (-1); } - // Run as data reader + // Run as data reader - ret = test_sub_run(dims, np, - sp, timestep, elem_size, num_vars, terminate, gcomm); + ret = test_sub_run(dims, np, sp, timestep, elem_size, num_vars, terminate, + gcomm); - MPI_Barrier(gcomm); - MPI_Finalize(); + MPI_Barrier(gcomm); + MPI_Finalize(); if(ret) { goto err_out; @@ -154,8 +165,8 @@ int main(int argc, char **argv) fprintf(stderr, "That's all from test_sub, folks!\n"); } - return ret; + return ret; err_out: - fprintf(stderr, "test_sub rank %d has failed!\n", rank); - return -1; + fprintf(stderr, "test_sub rank %d has failed!\n", rank); + return -1; } diff --git a/tests/test_sub_run.c b/tests/test_sub_run.c index a7df5956..250c2da9 100644 --- a/tests/test_sub_run.c +++ b/tests/test_sub_run.c @@ -1,21 +1,22 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ +#include "mpi.h" +#include "timer.h" +#include #include +#include #include #include #include -#include -#include -#include "timer.h" -#include "mpi.h" //# of processors in fortran direction static int np[10] = {0}; -//block size per processor per direction +// block size per processor per direction static uint64_t sp[10] = {0}; //# of interations static int timesteps_; @@ -34,59 +35,61 @@ static size_t elem_size_; static char transport_type_str_[256]; -static double* allocate_nd(int dims) +static double *allocate_nd(int dims) { - double* tmp = NULL; - int i = 0; - uint64_t size = 1; - for(i = 0; i < dims; i++){ - size *= sp[i]; - } - tmp = (double*)malloc(elem_size_ * size); - return tmp; + double *tmp = NULL; + int i = 0; + uint64_t size = 1; + for(i = 0; i < dims; i++) { + size *= sp[i]; + } + tmp = (double *)malloc(elem_size_ * size); + return tmp; } static void set_offset_nd(int rank, int dims) { - int i = 0, j = 0; - for(i = 0; i < dims; i++){ - int tmp = rank; - for(j = 0; j < i; j++) - tmp /= np[j]; - off[i] = tmp % np[i] * sp[i]; - } + int i = 0, j = 0; + for(i = 0; i < dims; i++) { + int tmp = rank; + for(j = 0; j < i; j++) + tmp /= np[j]; + off[i] = tmp % np[i] * sp[i]; + } } -int check_data(const char *var_name, double *buf, int num_elem, int rank, int ts) +int check_data(const char *var_name, double *buf, int num_elem, int rank, + int ts) { - double max, min, sum, avg; - int i; - int cnt = 0; + double max, min, sum, avg; + int i; + int cnt = 0; - if (num_elem <= 0) { - return -EINVAL; - } - max = min = sum = buf[0]; - for (i = 0; i < num_elem; i++) { - if (max < buf[i]) - max = buf[i]; - if (min > buf [i]) - min = buf[i]; - sum += buf[i]; - if (buf[i] != ts) { - cnt++; - } + if(num_elem <= 0) { + return -EINVAL; + } + max = min = sum = buf[0]; + for(i = 0; i < num_elem; i++) { + if(max < buf[i]) + max = buf[i]; + if(min > buf[i]) + min = buf[i]; + sum += buf[i]; + if(buf[i] != ts) { + cnt++; } - avg = sum / num_elem; - if (cnt > 0) { - fprintf(stderr, "%s(): var= %s, rank= %d, ts= %d, " + } + avg = sum / num_elem; + if(cnt > 0) { + fprintf(stderr, + "%s(): var= %s, rank= %d, ts= %d, " "error elem cnt= %d, total elem= %d\n", - __func__, var_name, rank, ts, cnt, num_elem); - } + __func__, var_name, rank, ts, cnt, num_elem); + } - free(buf); + free(buf); - return cnt; + return cnt; } int check_data_cb(dspaces_client_t client, struct dspaces_req *req, void *rankv) @@ -95,21 +98,22 @@ int check_data_cb(dspaces_client_t client, struct dspaces_req *req, void *rankv) int num_elem = 1; int i; - fprintf(stderr, "executing %s on rank %d for version %d.\n", __func__, rank, req->ver); + fprintf(stderr, "executing %s on rank %d for version %d.\n", __func__, rank, + req->ver); for(i = 0; i < req->ndim; i++) { num_elem *= (req->ub[i] - req->lb[i]) + 1; } - return(check_data(req->var_name, req->buf, num_elem, rank, req->ver)); - + return (check_data(req->var_name, req->buf, num_elem, rank, req->ver)); } -static int couple_sub_nd(dspaces_client_t client, unsigned int ts, int num_vars, int dims, dspaces_sub_t *subh) +static int couple_sub_nd(dspaces_client_t client, unsigned int ts, int num_vars, + int dims, dspaces_sub_t *subh) { - double **data_tab = (double **)malloc(sizeof(double*) * num_vars); - char var_name[128]; - int i; + double **data_tab = (double **)malloc(sizeof(double *) * num_vars); + char var_name[128]; + int i; int ret = 0; int err = 0; uint64_t dims_size = 1; @@ -118,36 +122,36 @@ static int couple_sub_nd(dspaces_client_t client, unsigned int ts, int num_vars, double tm_st, tm_end, tm_max, tm_diff; int root = 0; - for(i = 0; i < num_vars; i++){ - data_tab[i] = NULL; - } + for(i = 0; i < num_vars; i++) { + data_tab[i] = NULL; + } - set_offset_nd(rank_, dims); - for(i = 0; i < dims; i++){ - lb[i] = off[i]; - ub[i] = off[i] + sp[i] - 1; - dims_size *= sp[i]; - } + set_offset_nd(rank_, dims); + for(i = 0; i < dims; i++) { + lb[i] = off[i]; + ub[i] = off[i] + sp[i] - 1; + dims_size *= sp[i]; + } - MPI_Barrier(gcomm_); + MPI_Barrier(gcomm_); tm_st = timer_read(&timer_); - for(i = 0; i < num_vars; i++){ - sprintf(var_name, "mnd_%d", i); - *subh = dspaces_sub(client, var_name, ts, elem_size, dims, lb, ub, check_data_cb, &rank_); - if(*subh == DSPACES_SUB_FAIL) { + for(i = 0; i < num_vars; i++) { + sprintf(var_name, "mnd_%d", i); + *subh = dspaces_sub(client, var_name, ts, elem_size, dims, lb, ub, + check_data_cb, &rank_); + if(*subh == DSPACES_SUB_FAIL) { fprintf(stderr, "dspaces_sub() failed.\n"); - return(-1); + return (-1); } - } - tm_end = timer_read(&timer_); - - tm_diff = tm_end-tm_st; - MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); + } + tm_end = timer_read(&timer_); - if (rank_ == root) { - fprintf(stdout, "TS= %u read MAX time= %lf\n", - ts, tm_max); + tm_diff = tm_end - tm_st; + MPI_Reduce(&tm_diff, &tm_max, 1, MPI_DOUBLE, MPI_MAX, root, gcomm_); + + if(rank_ == root) { + fprintf(stdout, "TS= %u read MAX time= %lf\n", ts, tm_max); } free(data_tab); @@ -155,34 +159,33 @@ static int couple_sub_nd(dspaces_client_t client, unsigned int ts, int num_vars, return ret; } -int test_sub_run(int ndims, int* npdim, - uint64_t *spdim, int timestep, size_t elem_size, int num_vars, int terminate, - MPI_Comm gcomm) +int test_sub_run(int ndims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int terminate, MPI_Comm gcomm) { dspaces_sub_t *sub_handles; - gcomm_ = gcomm; - elem_size_ = elem_size; - timesteps_ = timestep; + gcomm_ = gcomm; + elem_size_ = elem_size; + timesteps_ = timestep; int result; - dspaces_client_t ndcl = dspaces_CLIENT_NULL; + dspaces_client_t ndcl = dspaces_CLIENT_NULL; - hg_return_t hret = HG_SUCCESS; + hg_return_t hret = HG_SUCCESS; int err, ret = 0; - int i; - for(i = 0; i < ndims; i++){ + int i; + for(i = 0; i < ndims; i++) { np[i] = npdim[i]; sp[i] = spdim[i]; - } + } - timer_init(&timer_, 1); + timer_init(&timer_, 1); timer_start(&timer_); - double tm_st, tm_end; - tm_st = timer_read(&timer_); - - MPI_Comm_rank(gcomm_, &rank_); + double tm_st, tm_end; + tm_st = timer_read(&timer_); + + MPI_Comm_rank(gcomm_, &rank_); MPI_Comm_size(gcomm_, &nproc_); ret = dspaces_init(rank_, &ndcl); @@ -190,34 +193,37 @@ int test_sub_run(int ndims, int* npdim, fprintf(stderr, "%s: dspaces_init() failed with %d.\n", __func__, ret); } - tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); - + tm_end = timer_read(&timer_); + fprintf(stdout, "TIMING_PERF Init_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); + sub_handles = malloc(sizeof(*sub_handles) * timesteps_); - unsigned int ts; - for(ts = 1; ts <= timesteps_; ts++) { - err = couple_sub_nd(ndcl, ts, num_vars, ndims, &sub_handles[ts-1]); - if(err != 0) { - fprintf(stderr, "couple_sub_nd failed on ts %d with %d.\n", ts, err); - ret = -1; - } - } + unsigned int ts; + for(ts = 1; ts <= timesteps_; ts++) { + err = couple_sub_nd(ndcl, ts, num_vars, ndims, &sub_handles[ts - 1]); + if(err != 0) { + fprintf(stderr, "couple_sub_nd failed on ts %d with %d.\n", ts, + err); + ret = -1; + } + } for(ts = 1; ts <= timesteps_; ts++) { - err = dspaces_check_sub(ndcl, sub_handles[ts-1], 1, &result); + err = dspaces_check_sub(ndcl, sub_handles[ts - 1], 1, &result); if((err != DSPACES_SUB_DONE) || result > 0) { - fprintf(stderr, "subscription tailed for ts %d with %d.\n", ts, err); + fprintf(stderr, "subscription tailed for ts %d with %d.\n", ts, + err); ret = -1; } - } - - MPI_Barrier(gcomm_); + } - if(rank_ == 0) { - fprintf(stdout, "%s(): done\n", __func__); - } - tm_st = timer_read(&timer_); + MPI_Barrier(gcomm_); + + if(rank_ == 0) { + fprintf(stdout, "%s(): done\n", __func__); + } + tm_st = timer_read(&timer_); if(rank_ == 0 && terminate) { fprintf(stderr, "Subscriber sending kill signal to server.\n"); @@ -225,15 +231,15 @@ int test_sub_run(int ndims, int* npdim, } dspaces_fini(ndcl); - tm_end = timer_read(&timer_); + tm_end = timer_read(&timer_); - fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", rank_, tm_end-tm_st); + fprintf(stdout, "TIMING_PERF Close_server_connection peer %d time= %lf\n", + rank_, tm_end - tm_st); return ret; - error: +error: dspaces_fini(ndcl); - + return ret; - } diff --git a/tests/test_writer.c b/tests/test_writer.c index 2636aa38..4abbcb83 100644 --- a/tests/test_writer.c +++ b/tests/test_writer.c @@ -1,36 +1,47 @@ /* - * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers University + * Copyright (c) 2020, Rutgers Discovery Informatics Institute, Rutgers + * University * * See COPYRIGHT in top-level directory. */ - -#include +#include "mpi.h" #include +#include #include #include -#include "mpi.h" -extern int test_put_run(int dims, - int* npdim, uint64_t *spdim, int timestep, - size_t elem_size, int num_vars, int local_mode, int terminate, MPI_Comm gcomm); +extern int test_put_run(int dims, int *npdim, uint64_t *spdim, int timestep, + size_t elem_size, int num_vars, int local_mode, + int terminate, MPI_Comm gcomm); void print_usage() { - fprintf(stderr, "Usage: test_writer np[0] .. np[dims-1] sp[0] ... sp[dims-1] [-s ] [-m (server|local)] [-c ] [-t]\n" -" dims - number of data dimensions. Must be at least one\n" -" np[i] - the number of processes in the ith dimension. The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" -" sp[i] - the per-process data size in the ith dimension\n" -" timesteps - the number of timestep iterations written\n" -" -s - the number of bytes in each element. Defaults to 8\n" -" -m (server|local) - the storage mode (stage to server or stage in process memory). Defaults to server\n" -" -c - the number of variables written in each iteration. Defaults to one\n" -" -t - send server termination after writing is complete\n"); + fprintf( + stderr, + "Usage: test_writer np[0] .. np[dims-1] sp[0] ... sp[dims-1] " + " [-s ] [-m (server|local)] [-c ] " + "[-t]\n" + " dims - number of data dimensions. Must be at least " + "one\n" + " np[i] - the number of processes in the ith dimension. " + "The product of np[0],...,np[dim-1] must be the number of MPI ranks\n" + " sp[i] - the per-process data size in the ith " + "dimension\n" + " timesteps - the number of timestep iterations written\n" + " -s - the number of bytes in each element. Defaults " + "to 8\n" + " -m (server|local) - the storage mode (stage to server or stage in " + "process memory). Defaults to server\n" + " -c - the number of variables written in each " + "iteration. Defaults to one\n" + " -t - send server termination after writing is " + "complete\n"); } -int parse_args(int argc, char **argv, - int *dims, int *npdim, uint64_t *spdim, int *timestep, - size_t *elem_size, int *num_vars, int *store_local, int *terminate) +int parse_args(int argc, char **argv, int *dims, int *npdim, uint64_t *spdim, + int *timestep, size_t *elem_size, int *num_vars, + int *store_local, int *terminate) { char **argp; int i; @@ -41,17 +52,17 @@ int parse_args(int argc, char **argv, *dims = 1; *terminate = 0; if(argc > 1) { - *dims = atoi(argv[1]); + *dims = atoi(argv[1]); } if(argc < 3 + (*dims * 2)) { fprintf(stderr, "Not enough arguments.\n"); print_usage(); - return(-1); + return (-1); } argp = &argv[2]; - + for(i = 0; i < *dims; i++) { npdim[i] = atoi(*argp); argp++; @@ -69,35 +80,35 @@ int parse_args(int argc, char **argv, if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *elem_size = atoi(*(argp+1)); + *elem_size = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-m") == 0) { if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - if(strcmp(*(argp+1), "local") == 0) { + if(strcmp(*(argp + 1), "local") == 0) { *store_local = 1; } - argp += 2; + argp += 2; } else if(strcmp(*argp, "-c") == 0) { if(argp == ((argv + argc) - 1)) { fprintf(stderr, "%s takes an argument.\n", *argp); print_usage(); - return(-1); + return (-1); } - *num_vars = atoi(*(argp+1)); + *num_vars = atoi(*(argp + 1)); argp += 2; } else if(strcmp(*argp, "-t") == 0) { - *terminate = 1; + *terminate = 1; argp++; } else { fprintf(stderr, "Unknown argument: %s\n", *argp); print_usage(); - return(-1); + return (-1); } } @@ -105,67 +116,69 @@ int parse_args(int argc, char **argv, fprintf(stderr, "Warning: ignoring extraneous argument '%s'.\n", *argp); } - return(0); + return (0); } int main(int argc, char **argv) { - int err; - int nprocs, rank; - MPI_Comm gcomm; + int err; + int nprocs, rank; + MPI_Comm gcomm; int i; - int npapp; // number of application processes - int np[10] = {0}; //number of processes in each dimension - uint64_t sp[10] = {0}; //block size per process in each dimension - int timestep; // number of iterations - int dims; // number of dimensions - size_t elem_size; // Optional: size of one element in the global array. Default value is 8 (bytes). - int num_vars; // Optional: number of variables to be shared in the testing. Default value is 1. + int npapp; // number of application processes + int np[10] = {0}; // number of processes in each dimension + uint64_t sp[10] = {0}; // block size per process in each dimension + int timestep; // number of iterations + int dims; // number of dimensions + size_t elem_size; // Optional: size of one element in the global array. + // Default value is 8 (bytes). + int num_vars; // Optional: number of variables to be shared in the testing. + // Default value is 1. int local_mode; // Optional: local storage mode flag - int terminate; // Optional: send terminate to server flag + int terminate; // Optional: send terminate to server flag - if (parse_args(argc, argv, &dims, np, sp, - ×tep, &elem_size, &num_vars, &local_mode, &terminate) != 0) { - return(-1); - } + if(parse_args(argc, argv, &dims, np, sp, ×tep, &elem_size, &num_vars, + &local_mode, &terminate) != 0) { + return (-1); + } npapp = 1; for(i = 0; i < dims; i++) { npapp *= np[i]; } - // Using SPMD style programming - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Barrier(MPI_COMM_WORLD); - gcomm = MPI_COMM_WORLD; + // Using SPMD style programming + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Barrier(MPI_COMM_WORLD); + gcomm = MPI_COMM_WORLD; - int color = 1; - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); + int color = 1; + MPI_Comm_split(MPI_COMM_WORLD, color, rank, &gcomm); if(npapp != nprocs) { - fprintf(stderr, "Product of np[i] args must equal number of MPI processes!\n"); + fprintf(stderr, + "Product of np[i] args must equal number of MPI processes!\n"); print_usage(); - return(-1); + return (-1); } - // Run as data writer + // Run as data writer - test_put_run(dims, np, - sp, timestep, elem_size, num_vars, local_mode, terminate, gcomm); + test_put_run(dims, np, sp, timestep, elem_size, num_vars, local_mode, + terminate, gcomm); - MPI_Barrier(gcomm); - MPI_Finalize(); + MPI_Barrier(gcomm); + MPI_Finalize(); if(rank == 0) { fprintf(stderr, "That's all from test_writer, folks!\n"); - } + } - return 0; + return 0; err_out: - fprintf(stderr, "test_writer rank %d has failed.!\n", rank); - return -1; + fprintf(stderr, "test_writer rank %d has failed.!\n", rank); + return -1; } - diff --git a/tests/timer.c b/tests/timer.c index 4562af07..8434e290 100644 --- a/tests/timer.c +++ b/tests/timer.c @@ -1,160 +1,162 @@ /* - * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers University - * All rights reserved. + * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers + * University All rights reserved. * - * Redistribution and use in source and binary forms, with or without modification, are permitted provided - * that the following conditions are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * - Redistributions of source code must retain the above copyright notice, this list of conditions and - * the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and - * the following disclaimer in the documentation and/or other materials provided with the distribution. - * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers University, nor the names of its - * contributors may be used to endorse or promote products derived from this software without specific prior - * written permission. + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers + * University, nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written + * permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* -* Ciprian Docan (2009) TASSL Rutgers University -* docan@cac.rutgers.edu -*/ + * Ciprian Docan (2009) TASSL Rutgers University + * docan@cac.rutgers.edu + */ #include "timer.h" #ifdef XT3 -static double tv_diff( double start, double end ) -{ - return end - start; -} +static double tv_diff(double start, double end) { return end - start; } #else -static double tv_diff( struct timeval* tv_start, struct timeval* tv_end ) +static double tv_diff(struct timeval *tv_start, struct timeval *tv_end) { - // returns the time difference in seconds - long sec, usec; + // returns the time difference in seconds + long sec, usec; - sec = tv_end->tv_sec - tv_start->tv_sec; - usec = tv_end->tv_usec - tv_start->tv_usec; - if( usec < 0 ) { - sec = sec - 1; - usec += 1000000; - } + sec = tv_end->tv_sec - tv_start->tv_sec; + usec = tv_end->tv_usec - tv_start->tv_usec; + if(usec < 0) { + sec = sec - 1; + usec += 1000000; + } - return (double) (sec + (double) usec / 1.e6); + return (double)(sec + (double)usec / 1.e6); } -#endif // XT3 +#endif // XT3 -static void __timer_update( mtimer_t* timer ) +static void __timer_update(mtimer_t *timer) { #ifdef XT3 - double now; + double now; - now = dclock(); - timer->elapsed_time += tv_diff( timer->stoptime, now ); - timer->stoptime = now; + now = dclock(); + timer->elapsed_time += tv_diff(timer->stoptime, now); + timer->stoptime = now; #else - struct timeval now; + struct timeval now; - gettimeofday( &now, 0 ); - timer->elapsed_time += tv_diff( &timer->stop_time, &now ); - timer->stop_time = now; - // gettimeofday( &timer->stop_time, 0 ); + gettimeofday(&now, 0); + timer->elapsed_time += tv_diff(&timer->stop_time, &now); + timer->stop_time = now; + // gettimeofday( &timer->stop_time, 0 ); #endif } -static void __timer_reset(mtimer_t* timer) +static void __timer_reset(mtimer_t *timer) { #ifdef XT3 - timer->starttime = timer->stoptime = dclock(); + timer->starttime = timer->stoptime = dclock(); #else - gettimeofday(&timer->start_time, 0); - timer->stop_time = timer->start_time; + gettimeofday(&timer->start_time, 0); + timer->stop_time = timer->start_time; #endif } -void timer_init(mtimer_t* timer, unsigned int alarm_time) +void timer_init(mtimer_t *timer, unsigned int alarm_time) { - // input parameter "alarm_time" is expressed in mili-seconds - timer->elapsed_time = 0.0; + // input parameter "alarm_time" is expressed in mili-seconds + timer->elapsed_time = 0.0; - timer->alarm_time = (double) alarm_time / 1.e3; - timer->stopped = 0; - timer->started = 0; + timer->alarm_time = (double)alarm_time / 1.e3; + timer->stopped = 0; + timer->started = 0; - __timer_reset( timer ); + __timer_reset(timer); } -void timer_start(mtimer_t* timer) +void timer_start(mtimer_t *timer) { - if (!timer->started) { - timer->started = 1; - if (!timer->stopped) - __timer_reset(timer); - else + if(!timer->started) { + timer->started = 1; + if(!timer->stopped) + __timer_reset(timer); + else #ifdef XT3 - timer->stoptime = dclock(); + timer->stoptime = dclock(); #else - gettimeofday(&timer->stop_time, 0); + gettimeofday(&timer->stop_time, 0); #endif - } - timer->stopped = 0; + } + timer->stopped = 0; } void timer_stop(mtimer_t *timer) { - if (!timer->stopped) { - __timer_update(timer); - timer->stopped = 1; - } - timer->started = 0; + if(!timer->stopped) { + __timer_update(timer); + timer->stopped = 1; + } + timer->started = 0; } void timer_reset(mtimer_t *timer) { - timer->elapsed_time = 0.0; + timer->elapsed_time = 0.0; - __timer_reset(timer); + __timer_reset(timer); } /* Function to read the elapsed time value. It does not stop the internal timer. */ double timer_read(mtimer_t *timer) { - if (timer->started && !timer->stopped) - __timer_update(timer); + if(timer->started && !timer->stopped) + __timer_update(timer); - return timer->elapsed_time; + return timer->elapsed_time; } -int timer_expired(mtimer_t* timer) +int timer_expired(mtimer_t *timer) { - if( !timer->stopped ) - __timer_update( timer ); + if(!timer->stopped) + __timer_update(timer); - return timer->alarm_time <= timer->elapsed_time; + return timer->alarm_time <= timer->elapsed_time; } /* Function to get and return the current (wall clock) time. */ double timer_timestamp(void) { - double ret; + double ret; #ifdef XT3 - ret = dclock(); + ret = dclock(); #else - struct timeval tv; + struct timeval tv; - gettimeofday( &tv, 0 ); - ret = (double) tv.tv_usec + tv.tv_sec * 1.e6; + gettimeofday(&tv, 0); + ret = (double)tv.tv_usec + tv.tv_sec * 1.e6; #endif - return ret; + return ret; } diff --git a/tests/timer.h b/tests/timer.h index c9542016..88619bcd 100644 --- a/tests/timer.h +++ b/tests/timer.h @@ -1,66 +1,71 @@ /* - * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers University - * All rights reserved. + * Copyright (c) 2009, NSF Cloud and Autonomic Computing Center, Rutgers + * University All rights reserved. * - * Redistribution and use in source and binary forms, with or without modification, are permitted provided - * that the following conditions are met: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * - Redistributions of source code must retain the above copyright notice, this list of conditions and - * the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and - * the following disclaimer in the documentation and/or other materials provided with the distribution. - * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers University, nor the names of its - * contributors may be used to endorse or promote products derived from this software without specific prior - * written permission. + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the NSF Cloud and Autonomic Computing Center, Rutgers + * University, nor the names of its contributors may be used to endorse or + * promote products derived from this software without specific prior written + * permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A - * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* -* Ciprian Docan (2009) TASSL Rutgers University -* docan@cac.rutgers.edu -*/ + * Ciprian Docan (2009) TASSL Rutgers University + * docan@cac.rutgers.edu + */ #ifndef __TIMER_H_ #define __TIMER_H_ #ifdef XT3 -# include +#include #else -# include +#include #endif struct timer { #ifdef XT3 - double starttime; - double stoptime; + double starttime; + double stoptime; #else - struct timeval start_time; - struct timeval stop_time; + struct timeval start_time; + struct timeval stop_time; #endif - double elapsed_time; - double alarm_time; + double elapsed_time; + double alarm_time; - unsigned char stopped:1; - unsigned char started:1; - unsigned char reset:1; + unsigned char stopped : 1; + unsigned char started : 1; + unsigned char reset : 1; }; typedef struct timer mtimer_t; -void timer_init( mtimer_t*, unsigned int ); -void timer_start( mtimer_t* ); -void timer_stop( mtimer_t* ); -void timer_reset( mtimer_t* ); -double timer_read( mtimer_t* ); -int timer_expired( mtimer_t* ); +void timer_init(mtimer_t *, unsigned int); +void timer_start(mtimer_t *); +void timer_stop(mtimer_t *); +void timer_reset(mtimer_t *); +double timer_read(mtimer_t *); +int timer_expired(mtimer_t *); double timer_timestamp(void); #endif