Skip to content

Commit

Permalink
Use binary for all point reads
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattjala
mattjala committed Mar 14, 2024
1 parent 4c7c85b commit 504402f
Showing 1 changed file with 40 additions and 63 deletions.
103 changes: 40 additions & 63 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -602,11 +602,7 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
* All or Hyperslab selection. Point selections are dealt with by POSTing the
* point list as JSON in the request body.
*/
if (H5S_SEL_ERROR ==
(transfer_info[i].u.read_info.sel_type = H5Sget_select_type(transfer_info[i].file_space_id)))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type");

is_transfer_binary = !is_variable_str && (H5S_SEL_POINTS != transfer_info[i].u.read_info.sel_type);
is_transfer_binary = !is_variable_str;

/* Follow the semantics for the use of H5S_ALL */
if (H5S_ALL == transfer_info[i].mem_space_id && H5S_ALL == transfer_info[i].file_space_id) {
Expand Down Expand Up @@ -652,6 +648,10 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
"can't convert dataspace selection to string representation");
} /* end else */

if (H5S_SEL_ERROR ==
(transfer_info[i].u.read_info.sel_type = H5Sget_select_type(transfer_info[i].file_space_id)))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type");

/* Verify that the number of selected points matches */
if ((mem_select_npoints = H5Sget_select_npoints(transfer_info[i].mem_space_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "memory dataspace is invalid");
Expand Down Expand Up @@ -715,37 +715,16 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
* added as a request parameter to the GET URL.
*/
if (H5S_SEL_POINTS == transfer_info[i].u.read_info.sel_type) {
/* As the dataspace-selection-to-string function is not designed to include the enclosing '{' and
* '}', since returning just the selection string to the user makes more sense if they are
* including more elements in their JSON, we have to wrap the selection body here before sending
* it off to cURL
*/

/* Ensure we have enough space to add the enclosing '{' and '}' */
if (NULL == (transfer_info[i].selection_body =
(char *)RV_realloc(transfer_info[i].selection_body, selection_body_len + 3)))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"can't reallocate space for point selection body");

/* Shift the whole string down by a byte */
memmove(transfer_info[i].selection_body + 1, transfer_info[i].selection_body,
selection_body_len + 1);

/* Add in the braces */
transfer_info[i].selection_body[0] = '{';
transfer_info[i].selection_body[selection_body_len + 1] = '}';
transfer_info[i].selection_body[selection_body_len + 2] = '\0';

/* Check to make sure that the size of the selection HTTP body can safely be cast to a curl_off_t
*/
if (sizeof(curl_off_t) < sizeof(size_t))
ASSIGN_TO_SMALLER_SIZE(transfer_info[i].u.read_info.post_len, curl_off_t,
selection_body_len + 2, size_t)
ASSIGN_TO_SMALLER_SIZE(transfer_info[i].u.read_info.post_len, curl_off_t, selection_body_len,
size_t)
else if (sizeof(curl_off_t) > sizeof(size_t))
transfer_info[i].u.read_info.post_len = (curl_off_t)(selection_body_len + 2);
transfer_info[i].u.read_info.post_len = (curl_off_t)(selection_body_len);
else
ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(transfer_info[i].u.read_info.post_len, curl_off_t,
selection_body_len + 2, size_t)
selection_body_len, size_t)

if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_POST, 1))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL,
Expand All @@ -763,8 +742,14 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL POST data size: %s",
transfer_info[i].curl_err_buf);

transfer_info[i].curl_headers =
curl_slist_append(transfer_info[i].curl_headers, "Content-Type: application/json");
if (!is_transfer_binary) {
transfer_info[i].curl_headers =
curl_slist_append(transfer_info[i].curl_headers, "Content-Type: application/json");
}
else {
transfer_info[i].curl_headers = curl_slist_append(transfer_info[i].curl_headers,
"Content-Type: application/octet-stream");
}

#ifdef RV_CONNECTOR_DEBUG
printf("-> Setup cURL to POST point list for dataset read\n\n");
Expand Down Expand Up @@ -3982,15 +3967,29 @@ RV_convert_dataspace_selection_to_string(hid_t space_id, char **selection_string
break;

case H5S_SEL_POINTS:
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL,
"point selections are unsupported as a HTTP request parameter");
hssize_t num_points;
size_t point_buf_size;
#ifdef RV_CONNECTOR_DEBUG
printf("-> Point selection\n\n");
#endif
/* Format the point selection as a binary buffer containing each coordinate index as an
* hsize_t */
if ((num_points = H5Sget_select_npoints(space_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get number of selected points");

point_buf_size = (size_t)(ndims * num_points) * sizeof(hsize_t);
CHECKED_REALLOC_NO_PTR(out_string, out_string_len, point_buf_size, H5E_DATASPACE, FAIL);

if (H5Sget_select_elem_pointlist(space_id, 0, (hsize_t)num_points, (hsize_t *)out_string) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve point list");

out_string_curr_pos = out_string + point_buf_size;
break;

case H5S_SEL_HYPERSLABS: {
#ifdef RV_CONNECTOR_DEBUG
printf("-> Hyperslab selection\n\n");
#endif

/* Format the hyperslab selection according to the 'select' request/query parameter.
* This is composed of N triplets, one for each dimension of the dataspace, and looks like:
*
Expand Down Expand Up @@ -4590,7 +4589,6 @@ rv_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hi
size_t file_type_size = 0;
size_t mem_type_size = 0;
hbool_t needs_tconv = FALSE;
// RV_tconv_reuse_t reuse = RV_TCONV_REUSE_NONE;
hbool_t fill_bkg = FALSE;

/* Used to scatter read data into user buffer */
Expand Down Expand Up @@ -4621,37 +4619,16 @@ rv_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hi
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection type for file space");

if ((H5T_REFERENCE != dtype_class) && (H5T_VLEN != dtype_class) && !is_variable_str) {
if (sel_type != H5S_SEL_POINTS) {
resp_info.buffer = resp_buffer.buffer;
}
else {
/* Server response is JSON instead of binary.
* Parse its 'value' field to a binary array to use for src_buf */
if (RV_parse_response(resp_buffer.buffer, (void *)&file_type_id, (void *)&json_buf,
RV_json_values_to_binary_callback) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't parse values");

resp_info.buffer = json_buf;
}
resp_info.buffer = resp_buffer.buffer;
}
else {
if (H5T_VLEN == dtype_class) {
if (sel_type == H5S_SEL_POINTS) {
/* Variable length data returned as JSON */
if (RV_parse_response(resp_buffer.buffer, (void *)&mem_type_id, &vlen_buf,
RV_json_values_to_binary_callback) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't parse vlen data");

resp_info.buffer = vlen_buf;
}
else {
/* Variable length data returned in "packed" binary form */
if (RV_unpack_vlen_data(resp_buffer.buffer, mem_type_id, (size_t)file_select_npoints,
&vlen_buf) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't unpack vlen data from server");
/* Variable length data returned in "packed" binary form */
if (RV_unpack_vlen_data(resp_buffer.buffer, mem_type_id, (size_t)file_select_npoints, &vlen_buf) <
0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't unpack vlen data from server");

resp_info.buffer = vlen_buf;
}
resp_info.buffer = vlen_buf;
}
else if (H5T_STD_REF_OBJ == mem_type_id) {
/* Convert the received binary buffer into a buffer of rest_obj_ref_t's */
Expand Down

0 comments on commit 504402f

Please sign in to comment.