From 2a0a9b0af8111f0f55a9a6828f18938a3e27472a Mon Sep 17 00:00:00 2001 From: Ed Date: Thu, 29 Aug 2024 10:06:22 -0600 Subject: [PATCH 1/4] made tst_h_par_compress work for zlib, zstd --- h5_test/tst_h_par_compress.c | 45 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index e634758032..7094b1a264 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -15,39 +15,45 @@ #include "err_macros.h" #include -#define FILE_NAME "tst_h_par_compress.h5" -#define VAR_NAME "HALs_memory" +#define FILE_NAME "tst_h_par.h5" +#define VAR_NAME "Athelstan" #define NDIMS 1 #define MILLION 1000000 #define DIM2_LEN 16000000 #define SC1 100000 /* slice count. */ - -/* The following code, when uncommented, adds szip testing for - * parallel I/O. However, this currently fails. I have a support - * request in to HDF5 about this. Ed 7/8/20 */ -/* #ifdef HAVE_H5Z_SZIP */ -/* #define NUM_COMPRESS_FILTERS 2 */ -/* #else */ -/* #define NUM_COMPRESS_FILTERS 1 */ -/* #endif /\* HAVE_H5Z_SZIP *\/ */ -#define NUM_COMPRESS_FILTERS 1 +#define H5Z_FILTER_DEFLATE 1 +#define H5Z_FILTER_ZSTD 32015 +#define H5Z_FILTER_SZIP 4 +#define MAX_NUM_FILTERS 3 int main(int argc, char **argv) { int cf; int p, my_rank; + int num_compress_filters; + char filter_name[MAX_NUM_FILTERS][NC_MAX_NAME + 1]; + int id[MAX_NUM_FILTERS]; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Comm_size(MPI_COMM_WORLD, &p); + strcpy(filter_name[0], "zlib"); + id[0] = H5Z_FILTER_DEFLATE; + num_compress_filters = 1; +#ifdef HAVE_ZSTD + id[num_compress_filters] = H5Z_FILTER_ZSTD; + strcpy(filter_name[num_compress_filters], "zstd"); + num_compress_filters++; +#endif + /* For builds with HDF5 prior to 1.10.3, just return success. */ #ifdef HDF5_SUPPORTS_PAR_FILTERS - for (cf = 0; cf < NUM_COMPRESS_FILTERS; cf++) + for (cf = 0; cf < num_compress_filters; cf++) { if (!my_rank) - printf("*** Testing parallel I/O with %s compression...", cf ? "szip" : "zlib"); + printf("*** Testing parallel I/O with %s compression...", filter_name[cf]); { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; hid_t plistid; @@ -56,6 +62,7 @@ main(int argc, char **argv) int data[SC1], data_in[SC1]; int num_steps; int deflate_level = 4; + unsigned int ulevel = 1; int i, s; /* We will write the same slice of random data over and over to @@ -87,15 +94,15 @@ main(int argc, char **argv) if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0) ERR; /* Set compression, either deflate or szip. */ - if (cf == 0) + if (id[cf] == H5Z_FILTER_DEFLATE) { if (H5Pset_deflate(plistid, deflate_level) < 0) ERR; } - else + else if (id[cf] == H5Z_FILTER_ZSTD) { - int options_mask = 32; - int bits_per_pixel = 32; - if (H5Pset_szip(plistid, options_mask, bits_per_pixel)) ERR; + herr_t code; + if ((code = H5Pset_filter(plistid, H5Z_FILTER_ZSTD, H5Z_FLAG_OPTIONAL, 1, &ulevel))) + ERR; } /* Set chunking. */ From 67240d61b6bf2d18fa0674a4ce95a7947b13282c Mon Sep 17 00:00:00 2001 From: Ed Date: Thu, 29 Aug 2024 10:15:20 -0600 Subject: [PATCH 2/4] now test with szip too --- h5_test/tst_h_par_compress.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 7094b1a264..dcdcbe2fd8 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -39,14 +39,24 @@ main(int argc, char **argv) MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); MPI_Comm_size(MPI_COMM_WORLD, &p); - strcpy(filter_name[0], "zlib"); + /* We always have zlib. */ id[0] = H5Z_FILTER_DEFLATE; + strcpy(filter_name[0], "zlib"); num_compress_filters = 1; + + /* We might have zstd. */ #ifdef HAVE_ZSTD id[num_compress_filters] = H5Z_FILTER_ZSTD; strcpy(filter_name[num_compress_filters], "zstd"); num_compress_filters++; -#endif +#endif + + /* We might have szip. */ +#ifdef HAVE_H5Z_SZIP + id[num_compress_filters] = H5Z_FILTER_SZIP; + strcpy(filter_name[num_compress_filters], "szip"); + num_compress_filters++; +#endif /* For builds with HDF5 prior to 1.10.3, just return success. */ #ifdef HDF5_SUPPORTS_PAR_FILTERS @@ -104,6 +114,12 @@ main(int argc, char **argv) if ((code = H5Pset_filter(plistid, H5Z_FILTER_ZSTD, H5Z_FLAG_OPTIONAL, 1, &ulevel))) ERR; } + else if (id[cf] == H5Z_FILTER_SZIP) + { + int options_mask = 32; + int bits_per_pixel = 32; + if (H5Pset_szip(plistid, options_mask, bits_per_pixel)) ERR; + } /* Set chunking. */ if (H5Pset_chunk(plistid, NDIMS, &chunksize) < 0) ERR; From a3143a59b0aa75356199e487d91c85164a404ddd Mon Sep 17 00:00:00 2001 From: Ed Date: Thu, 29 Aug 2024 10:21:48 -0600 Subject: [PATCH 3/4] now test with szip too --- h5_test/tst_h_par_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index dcdcbe2fd8..39db404d9e 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -15,7 +15,7 @@ #include "err_macros.h" #include -#define FILE_NAME "tst_h_par.h5" +#define FILE_NAME "tst_h_par_compress.h5" #define VAR_NAME "Athelstan" #define NDIMS 1 #define MILLION 1000000 From d92ce1ce636f10a5697df5acb96f4cb00ccee359 Mon Sep 17 00:00:00 2001 From: Ed Date: Thu, 29 Aug 2024 10:22:50 -0600 Subject: [PATCH 4/4] now test with szip too --- h5_test/tst_h_par_compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 39db404d9e..37d85f8496 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -18,7 +18,6 @@ #define FILE_NAME "tst_h_par_compress.h5" #define VAR_NAME "Athelstan" #define NDIMS 1 -#define MILLION 1000000 #define DIM2_LEN 16000000 #define SC1 100000 /* slice count. */ #define H5Z_FILTER_DEFLATE 1