Skip to content

Commit

Permalink
Support link names up to ~64Kb
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattjala
mattjala committed Dec 18, 2023
1 parent 218f579 commit cc2a6e5
Show file tree
Hide file tree
Showing 7 changed files with 469 additions and 607 deletions.
649 changes: 214 additions & 435 deletions src/rest_vol_attr.c
View file

Large diffs are not rendered by default.

91 changes: 60 additions & 31 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,9 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
size_t path_len = 0;
char *host_header = NULL;
char *create_request_body = NULL;
char request_url[URL_MAX_LENGTH];
int url_len = 0;
void *ret_value = NULL;
char *request_url = NULL;
int url_len = 0;
void *ret_value = NULL;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received dataset create call with following parameters:\n");
Expand Down Expand Up @@ -240,6 +240,9 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");

/* Redirect cURL from the base URL to "/datasets" to create the dataset */
if ((request_url = RV_malloc(strlen(base_URL) + strlen("/datasets") + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate space for request url");

if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datasets", base_URL)) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, NULL, "snprintf error");

Expand Down Expand Up @@ -308,6 +311,8 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
RV_free(create_request_body);
if (host_header)
RV_free(host_header);
if (request_url)
RV_free(request_url);

/* Clean up allocated dataset object if there was an issue */
if (new_dataset && !ret_value)
Expand Down Expand Up @@ -495,11 +500,13 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
{
H5T_class_t dtype_class;
hbool_t is_transfer_binary = FALSE;
hbool_t is_select_request = FALSE;
htri_t is_variable_str;
hssize_t file_select_npoints = 0;
hssize_t mem_select_npoints = 0;
size_t selection_body_len = 0;
size_t host_header_len = 0;
size_t request_url_size = 0;
int url_len = 0;
herr_t ret_value = SUCCEED;
CURL *curl_multi_handle = NULL;
Expand All @@ -518,9 +525,6 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac

transfer_info[i].curl_easy_handle = curl_easy_duphandle(curl);

if ((transfer_info[i].request_url = calloc(URL_MAX_LENGTH, sizeof(char))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to allocate memory for request URLs");

if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_WRITEFUNCTION,
H5_rest_curl_write_data_callback_no_global))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set up non global curl write callback: %s",
Expand Down Expand Up @@ -674,17 +678,22 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac
transfer_info[i].curl_headers,
is_transfer_binary ? "Accept: application/octet-stream" : "Accept: application/json");

is_select_request = is_transfer_binary && transfer_info[i].selection_body &&
(H5S_SEL_POINTS != transfer_info[i].u.read_info.sel_type);

request_url_size = strlen(base_URL) + strlen("/datasets/") + strlen(transfer_info[i].dataset->URI) +
strlen("/value") + 1;

if (is_select_request)
request_url_size += (strlen("?select=") + strlen(transfer_info[i].selection_body));

if ((transfer_info[i].request_url = RV_calloc(request_url_size)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/datasets/<id>/value" to get the dataset data values */
if ((url_len = snprintf(transfer_info[i].request_url, URL_MAX_LENGTH, "%s/datasets/%s/value%s%s",
base_URL, transfer_info[i].dataset->URI,
is_transfer_binary && transfer_info[i].selection_body &&
(H5S_SEL_POINTS != transfer_info[i].u.read_info.sel_type)
? "?select="
: "",
is_transfer_binary && transfer_info[i].selection_body &&
(H5S_SEL_POINTS != transfer_info[i].u.read_info.sel_type)
? transfer_info[i].selection_body
: "")) < 0)
base_URL, transfer_info[i].dataset->URI, is_select_request ? "?select=" : "",
is_select_request ? transfer_info[i].selection_body : "")) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
Expand Down Expand Up @@ -843,6 +852,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa
H5S_sel_type sel_type = H5S_SEL_ALL;
H5T_class_t dtype_class;
hbool_t is_transfer_binary = FALSE;
hbool_t is_select_request = FALSE;
htri_t contiguous = FALSE;
htri_t is_variable_str;
hssize_t mem_select_npoints = 0;
Expand All @@ -851,6 +861,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa
size_t host_header_len = 0;
size_t write_body_len = 0;
size_t selection_body_len = 0;
size_t request_url_size = 0;
char *selection_body = NULL;
int url_len = 0;
herr_t ret_value = SUCCEED;
Expand Down Expand Up @@ -887,9 +898,6 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa

transfer_info[i].curl_easy_handle = curl_easy_duphandle(curl);

if ((transfer_info[i].request_url = calloc(URL_MAX_LENGTH, sizeof(char))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "failed to allocate memory for request URLs");

if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_WRITEFUNCTION,
H5_rest_curl_write_data_callback_no_global))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set up non global curl write callback: %s",
Expand Down Expand Up @@ -1118,13 +1126,21 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa
transfer_info[i].curl_headers,
is_transfer_binary ? "Content-Type: application/octet-stream" : "Content-Type: application/json");

is_select_request = is_transfer_binary && selection_body && (H5S_SEL_POINTS != sel_type);

request_url_size = strlen(base_URL) + strlen("/datasets/") + strlen(transfer_info[i].dataset->URI) +
strlen("/value") + 1;

if (is_select_request)
request_url_size += (strlen("?select=") + strlen(selection_body));

if ((transfer_info[i].request_url = RV_calloc(request_url_size)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/datasets/<id>/value" to write the value out */
if ((url_len = snprintf(
transfer_info[i].request_url, URL_MAX_LENGTH, "%s/datasets/%s/value%s%s", base_URL,
transfer_info[i].dataset->URI,
is_transfer_binary && selection_body && (H5S_SEL_POINTS != sel_type) ? "?select=" : "",
is_transfer_binary && selection_body && (H5S_SEL_POINTS != sel_type) ? selection_body
: "")) < 0)
if ((url_len = snprintf(transfer_info[i].request_url, URL_MAX_LENGTH, "%s/datasets/%s/value%s%s",
base_URL, transfer_info[i].dataset->URI, is_select_request ? "?select=" : "",
is_select_request ? selection_body : "")) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
Expand Down Expand Up @@ -1315,7 +1331,7 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
H5VL_file_specific_args_t vol_flush_args;
size_t host_header_len = 0;
char *host_header = NULL;
char request_url[URL_MAX_LENGTH];
char *request_url = NULL;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received dataset get call with following parameters:\n");
Expand Down Expand Up @@ -1379,6 +1395,10 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
if (RV_file_specific((void *)dset->domain, &vol_flush_args, H5P_DEFAULT, NULL) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "can't flush datase's domain");

if ((request_url = RV_malloc(strlen(base_URL) + strlen("/datasets/") + strlen(dset->URI) +
strlen("?verbose=1") + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

/* Make GET request to dataset with 'verbose' parameter for HSDS. */
snprintf(request_url, URL_MAX_LENGTH, "%s%s%s%s", base_URL, "/datasets/", dset->URI,
"?verbose=1");
Expand Down Expand Up @@ -1438,6 +1458,9 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
curl_headers = NULL;
}

if (request_url)
RV_free(request_url);

RV_free(host_header);

PRINT_ERROR_STACK;
Expand Down Expand Up @@ -1465,11 +1488,11 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
char *host_header = NULL;
char *request_body = NULL;
char *request_body_shape = NULL;
char request_url[URL_MAX_LENGTH];
int url_len = 0;
hid_t new_dspace_id = H5I_INVALID_HID;
hsize_t *old_extent = NULL;
hsize_t *maxdims = NULL;
char *request_url = NULL;
int url_len = 0;
hid_t new_dspace_id = H5I_INVALID_HID;
hsize_t *old_extent = NULL;
hsize_t *maxdims = NULL;
upload_info uinfo;

#ifdef RV_CONNECTOR_DEBUG
Expand Down Expand Up @@ -1581,7 +1604,6 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
uinfo.bytes_sent = 0;

/* Target dataset's shape URL */
memset(request_url, 0, URL_MAX_LENGTH);

/* Set up curl request */
host_header_len = strlen(dset->domain->u.file.filepath_name) + strlen(host_string) + 1;
Expand All @@ -1599,6 +1621,10 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s", curl_err_buf);

if ((request_url = RV_malloc(strlen(base_URL) + strlen("/datasets/") + strlen(dset->URI) +
strlen("/shape") + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

if ((url_len =
snprintf(request_url, URL_MAX_LENGTH, "%s/datasets/%s/shape", base_URL, dset->URI)) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error");
Expand Down Expand Up @@ -1655,6 +1681,9 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
if (host_header)
RV_free(host_header);

if (request_url)
RV_free(request_url);

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
Expand Down
15 changes: 10 additions & 5 deletions src/rest_vol_datatype.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,11 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
char *datatype_body = NULL;
char *link_body = NULL;
char *path_dirname = NULL;
char request_url[URL_MAX_LENGTH];
int commit_request_len = 0;
int link_body_len = 0;
int url_len = 0;
void *ret_value = NULL;
char *request_url = NULL;
int commit_request_len = 0;
int link_body_len = 0;
int url_len = 0;
void *ret_value = NULL;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received datatype commit call with following parameters:\n");
Expand Down Expand Up @@ -243,6 +243,9 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");

if ((request_url = RV_malloc(strlen(base_URL) + strlen("/datatypes") + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, NULL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/datatypes" to commit the datatype */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datatypes", base_URL)) < 0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, NULL, "snprintf error");
Expand Down Expand Up @@ -313,6 +316,8 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
RV_free(datatype_body);
if (link_body)
RV_free(link_body);
if (request_url)
RV_free(request_url);

/* Clean up allocated datatype object if there was an issue */
if (new_datatype && !ret_value)
Expand Down
23 changes: 19 additions & 4 deletions src/rest_vol_group.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ RV_group_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name
char *path_dirname = NULL;
char *base64_plist_buffer = NULL;
char target_URI[URI_MAX_LENGTH];
char request_url[URL_MAX_LENGTH];
char *request_url = NULL;
int create_request_body_len = 0;
int url_len = 0;
void *binary_plist_buffer = NULL;
Expand Down Expand Up @@ -241,6 +241,9 @@ RV_group_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name
/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");

if ((request_url = (char *)RV_malloc(strlen(base_URL) + strlen("/groups") + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTALLOC, NULL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/groups" to create the group */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/groups", base_URL)) < 0)
FUNC_GOTO_ERROR(H5E_SYM, H5E_SYSERRSTR, NULL, "snprintf error");
Expand Down Expand Up @@ -307,6 +310,8 @@ RV_group_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name
RV_free(create_request_body);
if (host_header)
RV_free(host_header);
if (request_url)
RV_free(request_url);

/* Clean up allocated group object if there was an issue */
if (new_group && !ret_value)
Expand Down Expand Up @@ -487,9 +492,9 @@ RV_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req)
RV_object_t *loc_obj = (RV_object_t *)obj;
size_t host_header_len = 0;
char *host_header = NULL;
char request_url[URL_MAX_LENGTH];
int url_len = 0;
herr_t ret_value = SUCCEED;
char *request_url = NULL;
int url_len = 0;
herr_t ret_value = SUCCEED;

loc_info loc_info_out;
memset(&loc_info_out, 0, sizeof(loc_info));
Expand Down Expand Up @@ -527,6 +532,10 @@ RV_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req)
object_type_to_string(loc_obj->obj_type));
#endif

if ((request_url = RV_malloc(strlen(base_URL) + strlen("/groups/") +
strlen(loc_obj->URI) + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/groups/<id>" to get information about the group */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/groups/%s", base_URL,
loc_obj->URI)) < 0)
Expand Down Expand Up @@ -576,6 +585,10 @@ RV_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req)
object_type_to_string(obj_type));
#endif

if ((request_url =
RV_malloc(strlen(base_URL) + strlen("/groups/") + strlen(temp_URI) + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTALLOC, FAIL, "can't allocate space for request url");

/* Redirect cURL from the base URL to "/groups/<id>" to get information about the group */
if ((url_len =
snprintf(request_url, URL_MAX_LENGTH, "%s/groups/%s", base_URL, temp_URI)) < 0)
Expand Down Expand Up @@ -663,6 +676,8 @@ RV_group_get(void *obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req)

if (host_header)
RV_free(host_header);
if (request_url)
RV_free(request_url);

if (curl_headers) {
curl_slist_free_all(curl_headers);
Expand Down
Loading

0 comments on commit cc2a6e5

Please sign in to comment.