Skip to content

Commit

Permalink
adding zstandard to bm_file
Browse files Browse the repository at this point in the history
  • Loading branch information
edwardhartnett committed Dec 3, 2023
1 parent 848a735 commit 2980338
Showing 1 changed file with 15 additions and 8 deletions.
23 changes: 15 additions & 8 deletions nc_perf/bm_file.c
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
size_t *data_read_us, int *data_write_us, int *in_format, int use_par,
int par_access, float *num_bytes, int p, int my_rank,
int slow_count, int verbose, int use_scs, int endianness,
int convert_unlim, int zstandard)
int convert_unlim, int zstandard, int nsd)
{
int ncid_in, ncid_out;
int natts, nvars, ndims, unlimdimid;
Expand Down Expand Up @@ -578,6 +578,9 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
/* Create the output var. */
if (nc_def_var(ncid_out, name, xtype, ndims, dimids, &varid_out)) ERR;

if (nsd)
if (nc_def_var_quantize(ncid_out, varid_out, NC_QUANTIZE_GRANULARBR, nsd)) ERR;

/* Set the output endianness. For simplicity in this program,
* all vars get the same endianness. But there's no reason why
* this couldn't be varied from var to var, though it is hard to
Expand Down Expand Up @@ -793,7 +796,6 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,

#define NDIMS 3
#define MAX_DEFLATE 9
#define INPUT_FILE "/upc/share/testdata/nssl/mosaic3d_nc/tile1/20070803-2300.netcdf"
#define COLON ":"
#define COMMA ","

Expand All @@ -814,13 +816,14 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
[-l] Convert unlimited dimensions to fixed dimensions.\n\
[-e 1|2] Set the endianness of output (1=little 2=big).\n\
[-y] Use zstandard compression instead of zlib.\n\
[-q nsd] Quantize to NSD.\n\
file Name of netCDF file\n"

static void
usage(void)
{
fprintf(stderr, "bm_file -v [-s N]|[-t V:S:S:S -u V:C:C:C -r V:I:I:I] -o file_out -f N -h"
" -c V:C:C,V:C:C:C -d -m -p -i -e 1|2 -l -y file\n%s", USAGE);
" -c V:C:C,V:C:C:C -d -m -p -i -e 1|2 -l -y -q N file\n%s", USAGE);
}

int
Expand Down Expand Up @@ -854,6 +857,7 @@ main(int argc, char **argv)
float read_rate, write_rate, reread_rate;
int slow_count = 10, use_scs = 0;
int endianness = 0;
int nsd = 0;
float num_bytes = 0;
int p = 1, my_rank = 0;
int c;
Expand All @@ -872,7 +876,7 @@ main(int argc, char **argv)
for (i = 0; i < MAX_DIMS; i++)
vo[o1].chunksize[i] = 0;

while ((c = getopt(argc, argv, "vo:f:hc:dpms:it:u:r:e:l:y")) != EOF)
while ((c = getopt(argc, argv, "vo:f:hc:dpms:it:u:r:e:l:yq:")) != EOF)
switch(c)
{
case 'v':
Expand Down Expand Up @@ -1018,6 +1022,9 @@ main(int argc, char **argv)
case 'y':
zstandard++;
break;
case 'q':
sscanf(optarg, "%d", &nsd);
break;
case '?':
usage();
return 1;
Expand Down Expand Up @@ -1078,7 +1085,7 @@ main(int argc, char **argv)
if ((ret = copy_file(file_in, file_out, cmode, num_vo, vo, &meta_read_us, &meta_write_us,
&data_read_us, &data_write_us, &in_format, use_par, par_access,
&num_bytes, p, my_rank, slow_count, verbose, use_scs, endianness,
convert_unlim, zstandard)))
convert_unlim, zstandard, nsd)))
return ret;

/* If the user wants a double check, make sure the data in the new
Expand Down Expand Up @@ -1151,7 +1158,7 @@ main(int argc, char **argv)
if (use_par)
printf("num_proc, ");
printf("deflate, shuffle, chunksize[0], chunksize[1], chunksize[2], "
"chunksize[3]\n");
"chunksize[3], zstandard, nsd\n");
}

printf("%d, %d, %ld, %ld, %d, %d, %ld, %d, %d, ", in_format, out_format, file_size(file_in),
Expand All @@ -1166,8 +1173,8 @@ main(int argc, char **argv)
printf("%d, ", p);
for (o1 = 0; o1 < num_vo; o1++)
{
printf("%d, %d, %d, %d, %d, %d ", vo[o1].deflate_num, vo[o1].shuffle,
(int)vo[o1].chunksize[0], (int)vo[o1].chunksize[1], (int)vo[o1].chunksize[2], (int)vo[o1].chunksize[3]);
printf("%d, %d, %d, %d, %d, %d, %d, %d", vo[o1].deflate_num, vo[o1].shuffle,
(int)vo[o1].chunksize[0], (int)vo[o1].chunksize[1], (int)vo[o1].chunksize[2], (int)vo[o1].chunksize[3], zstandard, nsd);
if (o1 >= MAX_VO_PRINTED)
break;
if (o1 != num_vo - 1)
Expand Down

0 comments on commit 2980338

Please sign in to comment.