From b13fcf96e7293c28fdb724385e94c50c3f281c02 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 19:38:18 -0600 Subject: [PATCH 01/50] added tst_gfs_data1 --- nc_test4/CMakeLists.txt | 1 + nc_test4/Makefile.am | 2 +- nc_test4/run_par_test.sh.in | 4 + nc_test4/tst_gfs_data_1.c | 190 +++++++++++++++++++++++++++++++ nc_test4/tst_parallel_compress.c | 2 +- 5 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 nc_test4/tst_gfs_data_1.c diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 5dd5a85ca3..1587738933 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -92,6 +92,7 @@ IF(TEST_PARALLEL4) build_bin_test(tst_parallel5) build_bin_test(tst_parallel_zlib) build_bin_test(tst_parallel_compress) + build_bin_test(tst_gfs_data_1) build_bin_test(tst_nc4perf) build_bin_test(tst_mode) build_bin_test(tst_simplerw_coll_r) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 82287329c7..03e455ecd0 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -87,7 +87,7 @@ endif # BUILD_UTILITIES if TEST_PARALLEL4 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \ tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \ -tst_mode tst_parallel_zlib tst_parallel_compress +tst_mode tst_parallel_zlib tst_parallel_compress tst_gfs_data_1 TESTS += run_par_test.sh endif # TEST_PARALLEL4 diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in index 77f2e9676e..dfac5a98b8 100644 --- a/nc_test4/run_par_test.sh.in +++ b/nc_test4/run_par_test.sh.in @@ -59,4 +59,8 @@ if test "@HAS_PAR_FILTERS@" = "yes"; then echo "Parallel I/O more tests with zlib and szip (if present in HDF5)." @MPIEXEC@ -n 1 ./tst_parallel_compress @MPIEXEC@ -n 4 ./tst_parallel_compress + + echo + echo "Parallel I/O like the NOAA GFS writes and reads data." + @MPIEXEC@ -n 4 ./tst_gfs_data_1 fi diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c new file mode 100644 index 0000000000..14c864926f --- /dev/null +++ b/nc_test4/tst_gfs_data_1.c @@ -0,0 +1,190 @@ +/* + Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and + redistribution conditions. + + This program tests netcdf-4 parallel I/O using zlib, shuffle, and + fletcher32 fliters while writing with parallel I/O. This works for + HDF5-1.10.3 and later. In this case HDF5_SUPPORTS_PAR_FILTERS will + be defined during configure. + + If szip was built into HDF5, this test will also test parallel szip + writes. If szip is present, then HAVE_H5Z_SZIP will be defined in + config.h. + + Ed Hartnett, 12/19/2019 +*/ + +#include +#include "err_macros.h" +#include + +#define FILE_NAME "tst_gfs_data_1.nc" +#define NDIMS 3 +#define DIMSIZE 24 +#define QTR_DATA (DIMSIZE * DIMSIZE / 4) +#define NUM_PROC 4 +#define NUM_SLABS 10 +#define NUM_SHUFFLE_SETTINGS 2 +#ifdef HAVE_H5Z_SZIP +#define NUM_COMPRESSION_FILTERS 2 +#else +#define NUM_COMPRESSION_FILTERS 1 +#endif + +int +main(int argc, char **argv) +{ + /* MPI stuff. */ + int mpi_size, mpi_rank; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + /* Netcdf-4 stuff. */ + int ncid, v1id, dimids[NDIMS]; + size_t start[NDIMS], count[NDIMS]; + + int f, i, res; + int *slab_data; /* one slab */ + + /* Initialize MPI. */ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + /* Allocate data. */ + if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; + + /* Create phony data. We're going to write a 24x24 array of ints, + in 4 sets of 144. */ + for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) + slab_data[i] = mpi_rank; + + if (!mpi_rank) + printf("\n*** Testing parallel writes with compression filters.\n"); + { + int s; + for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) + { + for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) + { + if (!mpi_rank) + { + printf("*** testing simple write with %s shuffle %d...", + (f ? "szip" : "zlib"), s); + } + + /* nc_set_log_level(3); */ + /* Create a parallel netcdf-4 file. */ + if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; + + /* Create three dimensions. */ + if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR; + if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR; + if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR; + + /* Create one var. Turn on deflation. */ + if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later + * versions. */ + if (!f) + res = nc_def_var_deflate(ncid, 0, s, 1, 1); + else + { + res = nc_def_var_deflate(ncid, 0, s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, 0, 32, 32); + } +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + /* Setting fletcher32 only will work for HDF5-1.10.3 and later + * versions. */ + res = nc_def_var_fletcher32(ncid, 0, 1); +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + /* Write metadata to file. */ + if (nc_enddef(ncid)) ERR; + + /* Set up slab for this process. */ + start[0] = mpi_rank * DIMSIZE/mpi_size; + start[1] = 0; + count[0] = DIMSIZE/mpi_size; + count[1] = DIMSIZE; + count[2] = 1; + /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n", + mpi_rank, start[0], start[1], count[0], count[1]);*/ + + /* Should not be allowed to change access to independent, + * because filters are in use. */ + if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR; + + /* Write slabs of data. */ + for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) + if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR; + + /* Close the netcdf file. */ + if (nc_close(ncid)) ERR; + + /* Check file. */ + { + int shuffle_in, deflate_in, deflate_level_in; + int options_mask_in, pixels_per_block_in; + int *slab_data_in; + + /* Allocate data. */ + if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; + + /* Reopen the file for parallel access. */ + if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; + + /* Check state of compression. */ + if (!f) + { + if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; + if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; + if (!deflate_in || deflate_level_in != 1) ERR; + } + else + { + if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; + if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; + if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; + } + + /* Use parallel I/O to read the data. */ + for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) + { + if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; + for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) + if (slab_data_in[i] != mpi_rank) ERR; + } + + /* Close the netcdf file. */ + if (nc_close(ncid)) ERR; + + free(slab_data_in); + } + + if (!mpi_rank) + SUMMARIZE_ERR; + } /* next shuffle filter test */ + } /* next compression filter (zlib and szip) */ + free(slab_data); + } + + /* Shut down MPI. */ + MPI_Finalize(); + + if (!mpi_rank) + FINAL_RESULTS; + + return 0; +} diff --git a/nc_test4/tst_parallel_compress.c b/nc_test4/tst_parallel_compress.c index a621a47637..de9a80f8a7 100644 --- a/nc_test4/tst_parallel_compress.c +++ b/nc_test4/tst_parallel_compress.c @@ -18,7 +18,7 @@ #include "err_macros.h" #include -#define FILE_NAME "tst_parallel_zlib2.nc" +#define FILE_NAME "tst_parallel_compress.nc" #define NDIMS 3 #define DIMSIZE 24 #define QTR_DATA (DIMSIZE * DIMSIZE / 4) From 52b3053b5191989b735cdb93c0e2574a06672d3f Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 20:18:43 -0600 Subject: [PATCH 02/50] starting to modify test --- nc_test4/tst_gfs_data_1.c | 285 ++++++++++++++++++++++++-------------- 1 file changed, 183 insertions(+), 102 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 14c864926f..399ed48111 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -1,17 +1,12 @@ /* - Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and + Copyright 2020, UCAR/Unidata See COPYRIGHT file for copying and redistribution conditions. - This program tests netcdf-4 parallel I/O using zlib, shuffle, and - fletcher32 fliters while writing with parallel I/O. This works for - HDF5-1.10.3 and later. In this case HDF5_SUPPORTS_PAR_FILTERS will - be defined during configure. + This program tests netcdf-4 parallel I/O using the same access + pattern as is used by NOAA's GFS when writing and reading model + data. - If szip was built into HDF5, this test will also test parallel szip - writes. If szip is present, then HAVE_H5Z_SZIP will be defined in - config.h. - - Ed Hartnett, 12/19/2019 + Ed Hartnett, 6/28/20 */ #include @@ -40,10 +35,11 @@ main(int argc, char **argv) MPI_Info info = MPI_INFO_NULL; /* Netcdf-4 stuff. */ - int ncid, v1id, dimids[NDIMS]; - size_t start[NDIMS], count[NDIMS]; + int ncid/*, v1id , dimids[NDIMS] */; + /* size_t start[NDIMS], count[NDIMS]; */ - int f, i, res; + int f, i; + /* int res; */ int *slab_data; /* one slab */ /* Initialize MPI. */ @@ -76,102 +72,187 @@ main(int argc, char **argv) /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - - /* Create three dimensions. */ - if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR; - if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR; - if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR; - - /* Create one var. Turn on deflation. */ - if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR; + +/* ! Turn off fill mode. */ +/* call check(nf90_set_fill(ncid, NF90_NOFILL, oldMode)) */ + +/* ! Define dimension grid_xt. */ +/* call check(nf90_def_dim(ncid, trim(dim_name(1)), dim_len(1), dimid(1))) */ + +/* ! Define dimension grid_yt. */ +/* call check(nf90_def_dim(ncid, trim(dim_name(2)), dim_len(2), dimid(2))) */ + +/* ! Define variable grid_xt. */ +/* call check(nf90_def_var(ncid, trim(var_name(1)), var_type(1), dimids=(/dimid(1)/), varid=varid(1))) */ +/* call check(nf90_var_par_access(ncid, varid(1), NF90_INDEPENDENT)) */ + +/* ! Define variable lon. */ +/* call check(nf90_def_var(ncid, trim(var_name(2)), var_type(2), dimids=(/dimid(1), dimid(2)/), varid=varid(2))) */ +/* ! call check(nf90_var_par_access(ncid, varid(2), NF90_INDEPENDENT)) */ + +/* ! Define variable grid_yt. */ +/* call check(nf90_def_var(ncid, trim(var_name(3)), var_type(3), dimids=(/dimid(2)/), varid=varid(3))) */ +/* call check(nf90_var_par_access(ncid, varid(3), NF90_INDEPENDENT)) */ + +/* ! Define variable lat. */ +/* call check(nf90_def_var(ncid, trim(var_name(4)), var_type(4), dimids=(/dimid(1), dimid(2)/), varid=varid(4))) */ +/* call check(nf90_var_par_access(ncid, varid(4), NF90_INDEPENDENT)) */ + +/* ! Define dimension pfull. */ +/* call check(nf90_def_dim(ncid, trim(dim_name(3)), dim_len(3), dimid(3))) */ + +/* ! Define variable pfull and write data. */ +/* call check(nf90_def_var(ncid, trim(var_name(5)), var_type(5), dimids=(/dimid(3)/), varid=varid(5))) */ +/* call check(nf90_var_par_access(ncid, varid(5), NF90_INDEPENDENT)) */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(5), start=(/pfull_start/), count=(/pfull_loc_size/), values=value_pfull_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Define dimension phalf. */ +/* call check(nf90_def_dim(ncid, trim(dim_name(4)), dim_len(4), dimid(4))) */ + +/* ! Define variable phalf and write data. */ +/* call check(nf90_def_var(ncid, trim(var_name(6)), var_type(6), dimids=(/dimid(4)/), varid=varid(6))) */ +/* call check(nf90_var_par_access(ncid, varid(6), NF90_INDEPENDENT)) */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(6), start=(/phalf_start/), count=(/phalf_loc_size/), values=value_phalf_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Define dimension time. */ +/* call check(nf90_def_dim(ncid, trim(dim_name(5)), dim_len(5), dimid(5))) */ + +/* ! Define variable time and write data. */ +/* call check(nf90_def_var(ncid, trim(var_name(7)), var_type(7), dimids=(/dimid(5)/), varid=varid(7))) */ +/* call check(nf90_var_par_access(ncid, varid(7), NF90_INDEPENDENT)) */ +/* call check(nf90_enddef(ncid)) */ +/* ! In NOAA code, do all processors write the single time value? */ +/* if (my_rank .eq. 0) then */ +/* call check(nf90_put_var(ncid, varid(7), values=value_time)) */ +/* endif */ +/* call check(nf90_redef(ncid)) */ + +/* ! Write variable grid_xt data. */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(1), start=(/grid_xt_start/), count=(/grid_xt_loc_size/), values=value_grid_xt_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Write lon data. */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(2), start=(/lon_xt_start, lon_yt_start/), count=(/lon_xt_loc_size, lon_yt_loc_size/), & */ +/* values=value_lon_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Write grid_yt data. */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(3), start=(/grid_yt_start/), count=(/grid_yt_loc_size/), values=value_grid_yt_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Write lat data. */ +/* call check(nf90_enddef(ncid)) */ +/* call check(nf90_put_var(ncid, varid(4), start=(/lat_xt_start, lat_yt_start/), count=(/lat_xt_loc_size, lat_yt_loc_size/), & */ +/* values=value_lat_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Define variable clwmr and write data (?) */ +/* call check(nf90_def_var(ncid, trim(var_name(8)), var_type(8), dimids=(/dimid(1), dimid(2), dimid(3), dimid(5)/), & */ +/* varid=varid(8), shuffle=.true., deflate_level=ideflate)) */ +/* call check(nf90_var_par_access(ncid, varid(8), NF90_COLLECTIVE)) */ +/* call check(nf90_enddef(ncid)) */ +/* ! call check(nf90_put_var(ncid, varid(8), values=value_clwmr)) */ +/* call check(nf90_put_var(ncid, varid(8), start=(/lat_xt_start, lat_yt_start, pfull_start, 1/), & */ +/* count=(/lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1/), values=value_clwmr_loc)) */ +/* call check(nf90_redef(ncid)) */ + +/* ! Close the file. */ +/* call check(nf90_close(ncid)) */ /* Setting any filter only will work for HDF5-1.10.3 and later * versions. */ - if (!f) - res = nc_def_var_deflate(ncid, 0, s, 1, 1); - else - { - res = nc_def_var_deflate(ncid, 0, s, 0, 0); - if (!res) - res = nc_def_var_szip(ncid, 0, 32, 32); - } -#ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; -#else - if (res != NC_EINVAL) ERR; -#endif - - /* Setting fletcher32 only will work for HDF5-1.10.3 and later - * versions. */ - res = nc_def_var_fletcher32(ncid, 0, 1); -#ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; -#else - if (res != NC_EINVAL) ERR; -#endif - - /* Write metadata to file. */ - if (nc_enddef(ncid)) ERR; - - /* Set up slab for this process. */ - start[0] = mpi_rank * DIMSIZE/mpi_size; - start[1] = 0; - count[0] = DIMSIZE/mpi_size; - count[1] = DIMSIZE; - count[2] = 1; - /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n", - mpi_rank, start[0], start[1], count[0], count[1]);*/ - - /* Should not be allowed to change access to independent, - * because filters are in use. */ - if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR; - - /* Write slabs of data. */ - for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) - if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR; +/* if (!f) */ +/* res = nc_def_var_deflate(ncid, 0, s, 1, 1); */ +/* else */ +/* { */ +/* res = nc_def_var_deflate(ncid, 0, s, 0, 0); */ +/* if (!res) */ +/* res = nc_def_var_szip(ncid, 0, 32, 32); */ +/* } */ +/* #ifdef HDF5_SUPPORTS_PAR_FILTERS */ +/* if (res) ERR; */ +/* #else */ +/* if (res != NC_EINVAL) ERR; */ +/* #endif */ + +/* /\* Setting fletcher32 only will work for HDF5-1.10.3 and later */ +/* * versions. *\/ */ +/* res = nc_def_var_fletcher32(ncid, 0, 1); */ +/* #ifdef HDF5_SUPPORTS_PAR_FILTERS */ +/* if (res) ERR; */ +/* #else */ +/* if (res != NC_EINVAL) ERR; */ +/* #endif */ + + /* /\* Write metadata to file. *\/ */ + /* if (nc_enddef(ncid)) ERR; */ + + /* /\* Set up slab for this process. *\/ */ + /* start[0] = mpi_rank * DIMSIZE/mpi_size; */ + /* start[1] = 0; */ + /* count[0] = DIMSIZE/mpi_size; */ + /* count[1] = DIMSIZE; */ + /* count[2] = 1; */ + /* /\*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n", */ + /* mpi_rank, start[0], start[1], count[0], count[1]);*\/ */ + + /* /\* Should not be allowed to change access to independent, */ + /* * because filters are in use. *\/ */ + /* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR; */ + + /* /\* Write slabs of data. *\/ */ + /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ + /* if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR; */ /* Close the netcdf file. */ if (nc_close(ncid)) ERR; - /* Check file. */ - { - int shuffle_in, deflate_in, deflate_level_in; - int options_mask_in, pixels_per_block_in; - int *slab_data_in; - - /* Allocate data. */ - if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; - - /* Reopen the file for parallel access. */ - if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; - - /* Check state of compression. */ - if (!f) - { - if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; - if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; - if (!deflate_in || deflate_level_in != 1) ERR; - } - else - { - if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; - if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; - if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; - } - - /* Use parallel I/O to read the data. */ - for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) - { - if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; - for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) - if (slab_data_in[i] != mpi_rank) ERR; - } - - /* Close the netcdf file. */ - if (nc_close(ncid)) ERR; - - free(slab_data_in); - } + /* /\* Check file. *\/ */ + /* { */ + /* int shuffle_in, deflate_in, deflate_level_in; */ + /* int options_mask_in, pixels_per_block_in; */ + /* int *slab_data_in; */ + + /* /\* Allocate data. *\/ */ + /* if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ + + /* /\* Reopen the file for parallel access. *\/ */ + /* if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; */ + + /* /\* Check state of compression. *\/ */ + /* if (!f) */ + /* { */ + /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; */ + /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ + /* if (!deflate_in || deflate_level_in != 1) ERR; */ + /* } */ + /* else */ + /* { */ + /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; */ + /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ + /* if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; */ + /* } */ + + /* /\* Use parallel I/O to read the data. *\/ */ + /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ + /* { */ + /* if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; */ + /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ + /* if (slab_data_in[i] != mpi_rank) ERR; */ + /* } */ + + /* /\* Close the netcdf file. *\/ */ + /* if (nc_close(ncid)) ERR; */ + + /* free(slab_data_in); */ + /* } */ if (!mpi_rank) SUMMARIZE_ERR; From 03540e2fb0ce5a87cd9411e257047ff31b50827c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 20:20:17 -0600 Subject: [PATCH 03/50] starting to modify test --- nc_test4/tst_gfs_data_1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 399ed48111..6028f1eba4 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -73,8 +73,8 @@ main(int argc, char **argv) /* Create a parallel netcdf-4 file. */ if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; -/* ! Turn off fill mode. */ -/* call check(nf90_set_fill(ncid, NF90_NOFILL, oldMode)) */ + /* Turn off fill mode. */ + nc_set_fill(ncid, NC_NOFILL, oldMode); /* ! Define dimension grid_xt. */ /* call check(nf90_def_dim(ncid, trim(dim_name(1)), dim_len(1), dimid(1))) */ From 19de11b1dd35225bac0e9e7aea14491f9e13dccc Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 20:31:07 -0600 Subject: [PATCH 04/50] starting to develop test --- nc_test4/tst_gfs_data_1.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 6028f1eba4..59f0d8611d 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -14,8 +14,7 @@ #include #define FILE_NAME "tst_gfs_data_1.nc" -#define NDIMS 3 -#define DIMSIZE 24 +#define NDIM5 5 #define QTR_DATA (DIMSIZE * DIMSIZE / 4) #define NUM_PROC 4 #define NUM_SLABS 10 @@ -34,13 +33,18 @@ main(int argc, char **argv) MPI_Comm comm = MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; - /* Netcdf-4 stuff. */ - int ncid/*, v1id , dimids[NDIMS] */; + int ncid; /* size_t start[NDIMS], count[NDIMS]; */ - int f, i; + /* Dimensions */ + char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; + int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; + int dimid[NDIM5]; + + int f; + /* int i; */ /* int res; */ - int *slab_data; /* one slab */ + /* int *slab_data; /\* one slab *\/ */ /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -48,12 +52,12 @@ main(int argc, char **argv) MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); /* Allocate data. */ - if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; + /* if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ /* Create phony data. We're going to write a 24x24 array of ints, in 4 sets of 144. */ - for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) - slab_data[i] = mpi_rank; + /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ + /* slab_data[i] = mpi_rank; */ if (!mpi_rank) printf("\n*** Testing parallel writes with compression filters.\n"); @@ -74,16 +78,16 @@ main(int argc, char **argv) if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; /* Turn off fill mode. */ - nc_set_fill(ncid, NC_NOFILL, oldMode); + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; -/* ! Define dimension grid_xt. */ -/* call check(nf90_def_dim(ncid, trim(dim_name(1)), dim_len(1), dimid(1))) */ + /* Define dimension grid_xt. */ + if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; -/* ! Define dimension grid_yt. */ -/* call check(nf90_def_dim(ncid, trim(dim_name(2)), dim_len(2), dimid(2))) */ + /* Define dimension grid_yt. */ + if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; -/* ! Define variable grid_xt. */ -/* call check(nf90_def_var(ncid, trim(var_name(1)), var_type(1), dimids=(/dimid(1)/), varid=varid(1))) */ + /* Define variable grid_xt. */ + /* if (nc_def_var(ncid, var_name[0], var_type(0), dimids=(/dimid(0)/), varid=varid(0))) ERR; */ /* call check(nf90_var_par_access(ncid, varid(1), NF90_INDEPENDENT)) */ /* ! Define variable lon. */ @@ -258,7 +262,7 @@ main(int argc, char **argv) SUMMARIZE_ERR; } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ - free(slab_data); + /* free(slab_data); */ } /* Shut down MPI. */ From 50eb14dbc7bf61cfc86948e37eee6ce1a55ed204 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 20:45:39 -0600 Subject: [PATCH 05/50] more test development --- nc_test4/tst_gfs_data_1.c | 132 +++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 53 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 59f0d8611d..34fde32d4b 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -18,9 +18,11 @@ #define QTR_DATA (DIMSIZE * DIMSIZE / 4) #define NUM_PROC 4 #define NUM_SLABS 10 -#define NUM_SHUFFLE_SETTINGS 2 +#define NUM_SHUFFLE_SETTINGS 1 +/* #define NUM_SHUFFLE_SETTINGS 2 */ #ifdef HAVE_H5Z_SZIP -#define NUM_COMPRESSION_FILTERS 2 +#define NUM_COMPRESSION_FILTERS 1 +/* #define NUM_COMPRESSION_FILTERS 2 */ #else #define NUM_COMPRESSION_FILTERS 1 #endif @@ -36,10 +38,34 @@ main(int argc, char **argv) int ncid; /* size_t start[NDIMS], count[NDIMS]; */ - /* Dimensions */ + /* Dimensions. */ char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; int dimid[NDIM5]; + int def_dimid[NDIM5]; + + /* Variables. */ +#define NUM_VARS 8 + char var_name[NUM_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", "lat", "pfull", "phalf", "time", "clwmr"}; + int varid[NUM_VARS]; + int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; + /* integer :: var_ndims(NUM_VARS) = (/ 1, 2, 1, 2, 1, 1, 1, 4 /) */ + /* integer :: ideflate = 4 */ + /* real*8 :: value_time = 2.0, value_time_in */ + /* real, allocatable :: value_clwmr(:,:,:,:) */ + /* integer :: phalf_loc_size, phalf_start */ + /* real, allocatable :: value_phalf_loc(:), value_phalf_loc_in(:) */ + /* integer :: pfull_loc_size, pfull_start */ + /* real, allocatable :: value_pfull_loc(:), value_pfull_loc_in(:) */ + /* integer :: grid_xt_loc_size, grid_xt_start */ + /* real, allocatable :: value_grid_xt_loc(:), value_grid_xt_loc_in(:) */ + /* integer :: grid_yt_loc_size, grid_yt_start */ + /* real, allocatable :: value_grid_yt_loc(:), value_grid_yt_loc_in(:) */ + /* integer :: lon_xt_loc_size, lon_xt_start, lon_yt_loc_size, lon_yt_start */ + /* real, allocatable :: value_lon_loc(:,:), value_lon_loc_in(:,:) */ + /* integer :: lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start */ + /* real, allocatable :: value_lat_loc(:,:), value_lat_loc_in(:,:) */ + /* real, allocatable :: value_clwmr_loc(:,:,:,:), value_clwmr_loc_in(:,:,:,:) */ int f; /* int i; */ @@ -87,88 +113,88 @@ main(int argc, char **argv) if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; /* Define variable grid_xt. */ - /* if (nc_def_var(ncid, var_name[0], var_type(0), dimids=(/dimid(0)/), varid=varid(0))) ERR; */ -/* call check(nf90_var_par_access(ncid, varid(1), NF90_INDEPENDENT)) */ + if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; + if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; -/* ! Define variable lon. */ -/* call check(nf90_def_var(ncid, trim(var_name(2)), var_type(2), dimids=(/dimid(1), dimid(2)/), varid=varid(2))) */ -/* ! call check(nf90_var_par_access(ncid, varid(2), NF90_INDEPENDENT)) */ + /* Define variable lon. */ + if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; + if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); -/* ! Define variable grid_yt. */ -/* call check(nf90_def_var(ncid, trim(var_name(3)), var_type(3), dimids=(/dimid(2)/), varid=varid(3))) */ -/* call check(nf90_var_par_access(ncid, varid(3), NF90_INDEPENDENT)) */ + /* Define variable grid_yt. */ + if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; + if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; -/* ! Define variable lat. */ -/* call check(nf90_def_var(ncid, trim(var_name(4)), var_type(4), dimids=(/dimid(1), dimid(2)/), varid=varid(4))) */ -/* call check(nf90_var_par_access(ncid, varid(4), NF90_INDEPENDENT)) */ + /* Define variable lat. */ + if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; + if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; -/* ! Define dimension pfull. */ -/* call check(nf90_def_dim(ncid, trim(dim_name(3)), dim_len(3), dimid(3))) */ + /* Define dimension pfull. */ + if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; -/* ! Define variable pfull and write data. */ -/* call check(nf90_def_var(ncid, trim(var_name(5)), var_type(5), dimids=(/dimid(3)/), varid=varid(5))) */ -/* call check(nf90_var_par_access(ncid, varid(5), NF90_INDEPENDENT)) */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(5), start=(/pfull_start/), count=(/pfull_loc_size/), values=value_pfull_loc)) */ -/* call check(nf90_redef(ncid)) */ + /* Define variable pfull and write data. */ + if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; + if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; +/* if (nc_put_var(ncid, varid(5), start=(/pfull_start/), count=(/pfull_loc_size/), values=value_pfull_loc)) */ +/* if (nc_redef(ncid)) */ /* ! Define dimension phalf. */ -/* call check(nf90_def_dim(ncid, trim(dim_name(4)), dim_len(4), dimid(4))) */ +/* if (nc_def_dim(ncid, trim(dim_name(4)), dim_len(4), dimid(4))) */ /* ! Define variable phalf and write data. */ -/* call check(nf90_def_var(ncid, trim(var_name(6)), var_type(6), dimids=(/dimid(4)/), varid=varid(6))) */ -/* call check(nf90_var_par_access(ncid, varid(6), NF90_INDEPENDENT)) */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(6), start=(/phalf_start/), count=(/phalf_loc_size/), values=value_phalf_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_def_var(ncid, trim(var_name(6)), var_type(6), dimids=(/dimid(4)/), varid=varid(6))) */ +/* if (nc_var_par_access(ncid, varid(6), NC_INDEPENDENT)) */ +/* if (nc_enddef(ncid)) */ +/* if (nc_put_var(ncid, varid(6), start=(/phalf_start/), count=(/phalf_loc_size/), values=value_phalf_loc)) */ +/* if (nc_redef(ncid)) */ /* ! Define dimension time. */ -/* call check(nf90_def_dim(ncid, trim(dim_name(5)), dim_len(5), dimid(5))) */ +/* if (nc_def_dim(ncid, trim(dim_name(5)), dim_len(5), dimid(5))) */ /* ! Define variable time and write data. */ -/* call check(nf90_def_var(ncid, trim(var_name(7)), var_type(7), dimids=(/dimid(5)/), varid=varid(7))) */ -/* call check(nf90_var_par_access(ncid, varid(7), NF90_INDEPENDENT)) */ -/* call check(nf90_enddef(ncid)) */ +/* if (nc_def_var(ncid, trim(var_name(7)), var_type(7), dimids=(/dimid(5)/), varid=varid(7))) */ +/* if (nc_var_par_access(ncid, varid(7), NC_INDEPENDENT)) */ +/* if (nc_enddef(ncid)) */ /* ! In NOAA code, do all processors write the single time value? */ /* if (my_rank .eq. 0) then */ -/* call check(nf90_put_var(ncid, varid(7), values=value_time)) */ +/* if (nc_put_var(ncid, varid(7), values=value_time)) */ /* endif */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_redef(ncid)) */ /* ! Write variable grid_xt data. */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(1), start=(/grid_xt_start/), count=(/grid_xt_loc_size/), values=value_grid_xt_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_enddef(ncid)) */ +/* if (nc_put_var(ncid, varid(1), start=(/grid_xt_start/), count=(/grid_xt_loc_size/), values=value_grid_xt_loc)) */ +/* if (nc_redef(ncid)) */ /* ! Write lon data. */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(2), start=(/lon_xt_start, lon_yt_start/), count=(/lon_xt_loc_size, lon_yt_loc_size/), & */ +/* if (nc_enddef(ncid)) */ +/* if (nc_put_var(ncid, varid(2), start=(/lon_xt_start, lon_yt_start/), count=(/lon_xt_loc_size, lon_yt_loc_size/), & */ /* values=value_lon_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_redef(ncid)) */ /* ! Write grid_yt data. */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(3), start=(/grid_yt_start/), count=(/grid_yt_loc_size/), values=value_grid_yt_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_enddef(ncid)) */ +/* if (nc_put_var(ncid, varid(3), start=(/grid_yt_start/), count=(/grid_yt_loc_size/), values=value_grid_yt_loc)) */ +/* if (nc_redef(ncid)) */ /* ! Write lat data. */ -/* call check(nf90_enddef(ncid)) */ -/* call check(nf90_put_var(ncid, varid(4), start=(/lat_xt_start, lat_yt_start/), count=(/lat_xt_loc_size, lat_yt_loc_size/), & */ +/* if (nc_enddef(ncid)) */ +/* if (nc_put_var(ncid, varid(4), start=(/lat_xt_start, lat_yt_start/), count=(/lat_xt_loc_size, lat_yt_loc_size/), & */ /* values=value_lat_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_redef(ncid)) */ /* ! Define variable clwmr and write data (?) */ -/* call check(nf90_def_var(ncid, trim(var_name(8)), var_type(8), dimids=(/dimid(1), dimid(2), dimid(3), dimid(5)/), & */ +/* if (nc_def_var(ncid, trim(var_name(8)), var_type(8), dimids=(/dimid(1), dimid(2), dimid(3), dimid(5)/), & */ /* varid=varid(8), shuffle=.true., deflate_level=ideflate)) */ -/* call check(nf90_var_par_access(ncid, varid(8), NF90_COLLECTIVE)) */ -/* call check(nf90_enddef(ncid)) */ -/* ! call check(nf90_put_var(ncid, varid(8), values=value_clwmr)) */ -/* call check(nf90_put_var(ncid, varid(8), start=(/lat_xt_start, lat_yt_start, pfull_start, 1/), & */ +/* if (nc_var_par_access(ncid, varid(8), NC_COLLECTIVE)) */ +/* if (nc_enddef(ncid)) */ +/* ! if (nc_put_var(ncid, varid(8), values=value_clwmr)) */ +/* if (nc_put_var(ncid, varid(8), start=(/lat_xt_start, lat_yt_start, pfull_start, 1/), & */ /* count=(/lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1/), values=value_clwmr_loc)) */ -/* call check(nf90_redef(ncid)) */ +/* if (nc_redef(ncid)) */ /* ! Close the file. */ -/* call check(nf90_close(ncid)) */ +/* if (nc_close(ncid)) */ /* Setting any filter only will work for HDF5-1.10.3 and later * versions. */ From 86e98892b6a172fe3f2a9d070739b7ce7c2715e2 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 20:56:02 -0600 Subject: [PATCH 06/50] more test development --- nc_test4/tst_gfs_data_1.c | 118 +++++++++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 7 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 34fde32d4b..8e22377418 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -14,6 +14,7 @@ #include #define FILE_NAME "tst_gfs_data_1.nc" +#define NDIM4 4 #define NDIM5 5 #define QTR_DATA (DIMSIZE * DIMSIZE / 4) #define NUM_PROC 4 @@ -36,13 +37,12 @@ main(int argc, char **argv) MPI_Info info = MPI_INFO_NULL; int ncid; - /* size_t start[NDIMS], count[NDIMS]; */ + size_t start[NDIM4], count[NDIM4]; /* Dimensions. */ char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; int dimid[NDIM5]; - int def_dimid[NDIM5]; /* Variables. */ #define NUM_VARS 8 @@ -55,8 +55,8 @@ main(int argc, char **argv) /* real, allocatable :: value_clwmr(:,:,:,:) */ /* integer :: phalf_loc_size, phalf_start */ /* real, allocatable :: value_phalf_loc(:), value_phalf_loc_in(:) */ - /* integer :: pfull_loc_size, pfull_start */ - /* real, allocatable :: value_pfull_loc(:), value_pfull_loc_in(:) */ + int pfull_loc_size, pfull_start; + float *value_pfull_loc, *value_pfull_loc_in; /* integer :: grid_xt_loc_size, grid_xt_start */ /* real, allocatable :: value_grid_xt_loc(:), value_grid_xt_loc_in(:) */ /* integer :: grid_yt_loc_size, grid_yt_start */ @@ -77,6 +77,92 @@ main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + /* ! Size of local (i.e. for this pe) grid_xt data. */ + /* grid_xt_loc_size = dim_len(1)/npes; */ + /* grid_xt_start = mpi_rank * grid_xt_loc_size + 1 */ + /* if (mpi_rank .eq. npes - 1) then */ + /* grid_xt_loc_size = grid_xt_loc_size + mod(dim_len(1), npes) */ + /* endif */ + /* !print *, mpi_rank, 'grid_xt', dim_len(3), grid_xt_start, grid_xt_loc_size */ + + /* ! Size of local (i.e. for this pe) grid_yt data. */ + /* grid_yt_loc_size = dim_len(2)/npes; */ + /* grid_yt_start = mpi_rank * grid_yt_loc_size + 1 */ + /* if (mpi_rank .eq. npes - 1) then */ + /* grid_yt_loc_size = grid_yt_loc_size + mod(dim_len(2), npes) */ + /* endif */ + /* !print *, mpi_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ + + /* Size of local (i.e. for this pe) pfull data. */ + pfull_loc_size = dim_len[2]/mpi_size; + pfull_start = mpi_rank * pfull_loc_size; + if (mpi_rank == mpi_size - 1) + pfull_loc_size = pfull_loc_size + dim_len[2] % mpi_size; + /* !print *, mpi_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ + + /* ! Size of local (i.e. for this pe) phalf data. */ + /* phalf_loc_size = dim_len(4)/npes; */ + /* phalf_start = mpi_rank * phalf_loc_size + 1 */ + /* if (mpi_rank .eq. npes - 1) then */ + /* phalf_loc_size = phalf_loc_size + mod(dim_len(4), npes) */ + /* endif */ + /* !print *, mpi_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ + + /* ! Size of local arrays (i.e. for this pe) lon and lat data. This is */ + /* ! specific to 4 pes. */ + /* lon_xt_loc_size = 1536 */ + /* lat_xt_loc_size = 1536 */ + /* if (mpi_rank .eq. 0 .or. mpi_rank .eq. 2) then */ + /* lon_xt_start = 1 */ + /* lat_xt_start = 1 */ + /* else */ + /* lon_xt_start = 1537 */ + /* lat_xt_start = 1537 */ + /* endif */ + /* lon_yt_loc_size = 768 */ + /* lat_yt_loc_size = 768 */ + /* if (mpi_rank .eq. 0 .or. mpi_rank .eq. 1) then */ + /* lon_yt_start = 1 */ + /* lat_yt_start = 1 */ + /* else */ + /* lon_yt_start = 769 */ + /* lat_yt_start = 769 */ + /* endif */ + /* ! print *, mpi_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ + /* ! print *, mpi_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ + + /* ! Allocate space on this pe to hold the data for this pe. */ + /* allocate(value_grid_xt_loc(grid_xt_loc_size)) */ + /* allocate(value_grid_xt_loc_in(grid_xt_loc_size)) */ + /* allocate(value_grid_yt_loc(grid_yt_loc_size)) */ + /* allocate(value_grid_yt_loc_in(grid_yt_loc_size)) */ + if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; + if (!(value_pfull_loc_in = malloc(pfull_loc_size * sizeof(float)))) ERR; + /* allocate(value_phalf_loc(phalf_loc_size)) */ + /* allocate(value_phalf_loc_in(phalf_loc_size)) */ + /* allocate(value_lon_loc(lon_xt_loc_size, lon_yt_loc_size)) */ + /* allocate(value_lon_loc_in(lon_xt_loc_size, lon_yt_loc_size)) */ + /* allocate(value_lat_loc(lat_xt_loc_size, lat_yt_loc_size)) */ + /* allocate(value_lat_loc_in(lat_xt_loc_size, lat_yt_loc_size)) */ + /* allocate(value_clwmr_loc(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ + /* allocate(value_clwmr_loc_in(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ + + /* ! Some fake data for this pe to write. */ + /* do i = 1, pfull_loc_size */ + /* value_pfull_loc(i) = mpi_rank * 100 + i; */ + /* end do */ + /* do i = 1, phalf_loc_size */ + /* value_phalf_loc(i) = mpi_rank * 100 + i; */ + /* end do */ + /* do i = 1, lon_xt_loc_size */ + /* do j = 1, lon_yt_loc_size */ + /* value_lon_loc(i, j) = mpi_rank * 100 + i +j */ + /* value_lat_loc(i, j) = mpi_rank * 100 + i +j */ + /* do k = 1, pfull_loc_size */ + /* value_clwmr_loc(i, j, k, 1) = mpi_rank * 100 + i + j + k */ + /* end do */ + /* end do */ + /* end do */ /* Allocate data. */ /* if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ @@ -135,8 +221,10 @@ main(int argc, char **argv) if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; -/* if (nc_put_var(ncid, varid(5), start=(/pfull_start/), count=(/pfull_loc_size/), values=value_pfull_loc)) */ -/* if (nc_redef(ncid)) */ + start[0] = pfull_start; + count[0] = pfull_loc_size; + if (nc_put_vara_float(ncid, varid[4], start, count, value_pfull_loc)) ERR; + if (nc_redef(ncid)) ERR; /* ! Define dimension phalf. */ /* if (nc_def_dim(ncid, trim(dim_name(4)), dim_len(4), dimid(4))) */ @@ -156,7 +244,7 @@ main(int argc, char **argv) /* if (nc_var_par_access(ncid, varid(7), NC_INDEPENDENT)) */ /* if (nc_enddef(ncid)) */ /* ! In NOAA code, do all processors write the single time value? */ -/* if (my_rank .eq. 0) then */ +/* if (mpi_rank .eq. 0) then */ /* if (nc_put_var(ncid, varid(7), values=value_time)) */ /* endif */ /* if (nc_redef(ncid)) */ @@ -289,6 +377,22 @@ main(int argc, char **argv) } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ /* free(slab_data); */ + + /* Free resources. */ + /* free(value_grid_xt_loc) */ + /* free(value_grid_xt_loc_in) */ + /* free(value_grid_yt_loc) */ + /* free(value_grid_yt_loc_in) */ + free(value_pfull_loc); + free(value_pfull_loc_in); + /* free(value_phalf_loc) */ + /* free(value_phalf_loc_in) */ + /* free(value_lon_loc) */ + /* free(value_lon_loc_in) */ + /* free(value_lat_loc) */ + /* free(value_lat_loc_in) */ + /* free(value_clwmr_loc) */ + /* free(value_clwmr_loc_in) */ } /* Shut down MPI. */ From 4413aba99f811d85e2779e26fe47a3f8e66f6e2e Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 28 Jun 2020 21:06:31 -0600 Subject: [PATCH 07/50] further test development --- nc_test4/tst_gfs_data_1.c | 63 ++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 8e22377418..009e5a5e15 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -37,7 +37,7 @@ main(int argc, char **argv) MPI_Info info = MPI_INFO_NULL; int ncid; - size_t start[NDIM4], count[NDIM4]; + /* size_t start[NDIM4], count[NDIM4]; */ /* Dimensions. */ char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -53,10 +53,10 @@ main(int argc, char **argv) /* integer :: ideflate = 4 */ /* real*8 :: value_time = 2.0, value_time_in */ /* real, allocatable :: value_clwmr(:,:,:,:) */ - /* integer :: phalf_loc_size, phalf_start */ - /* real, allocatable :: value_phalf_loc(:), value_phalf_loc_in(:) */ - int pfull_loc_size, pfull_start; + size_t pfull_loc_size, pfull_start; float *value_pfull_loc, *value_pfull_loc_in; + size_t phalf_loc_size, phalf_start; + float *value_phalf_loc, *value_phalf_loc_in; /* integer :: grid_xt_loc_size, grid_xt_start */ /* real, allocatable :: value_grid_xt_loc(:), value_grid_xt_loc_in(:) */ /* integer :: grid_yt_loc_size, grid_yt_start */ @@ -68,7 +68,7 @@ main(int argc, char **argv) /* real, allocatable :: value_clwmr_loc(:,:,:,:), value_clwmr_loc_in(:,:,:,:) */ int f; - /* int i; */ + int i; /* int res; */ /* int *slab_data; /\* one slab *\/ */ @@ -100,19 +100,18 @@ main(int argc, char **argv) pfull_loc_size = pfull_loc_size + dim_len[2] % mpi_size; /* !print *, mpi_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ - /* ! Size of local (i.e. for this pe) phalf data. */ - /* phalf_loc_size = dim_len(4)/npes; */ - /* phalf_start = mpi_rank * phalf_loc_size + 1 */ - /* if (mpi_rank .eq. npes - 1) then */ - /* phalf_loc_size = phalf_loc_size + mod(dim_len(4), npes) */ - /* endif */ + /* Size of local (i.e. for this pe) phalf data. */ + phalf_loc_size = dim_len[3]/mpi_size; + phalf_start = mpi_rank * phalf_loc_size; + if (mpi_rank == mpi_size - 1) + phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; /* !print *, mpi_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ /* ! Size of local arrays (i.e. for this pe) lon and lat data. This is */ /* ! specific to 4 pes. */ /* lon_xt_loc_size = 1536 */ /* lat_xt_loc_size = 1536 */ - /* if (mpi_rank .eq. 0 .or. mpi_rank .eq. 2) then */ + /* if (mpi_rank == 0 .or. mpi_rank == 2) then */ /* lon_xt_start = 1 */ /* lat_xt_start = 1 */ /* else */ @@ -121,7 +120,7 @@ main(int argc, char **argv) /* endif */ /* lon_yt_loc_size = 768 */ /* lat_yt_loc_size = 768 */ - /* if (mpi_rank .eq. 0 .or. mpi_rank .eq. 1) then */ + /* if (mpi_rank == 0 .or. mpi_rank == 1) then */ /* lon_yt_start = 1 */ /* lat_yt_start = 1 */ /* else */ @@ -138,8 +137,8 @@ main(int argc, char **argv) /* allocate(value_grid_yt_loc_in(grid_yt_loc_size)) */ if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; if (!(value_pfull_loc_in = malloc(pfull_loc_size * sizeof(float)))) ERR; - /* allocate(value_phalf_loc(phalf_loc_size)) */ - /* allocate(value_phalf_loc_in(phalf_loc_size)) */ + if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; + if (!(value_phalf_loc_in = malloc(phalf_loc_size * sizeof(float)))) ERR; /* allocate(value_lon_loc(lon_xt_loc_size, lon_yt_loc_size)) */ /* allocate(value_lon_loc_in(lon_xt_loc_size, lon_yt_loc_size)) */ /* allocate(value_lat_loc(lat_xt_loc_size, lat_yt_loc_size)) */ @@ -147,13 +146,11 @@ main(int argc, char **argv) /* allocate(value_clwmr_loc(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ /* allocate(value_clwmr_loc_in(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ - /* ! Some fake data for this pe to write. */ - /* do i = 1, pfull_loc_size */ - /* value_pfull_loc(i) = mpi_rank * 100 + i; */ - /* end do */ - /* do i = 1, phalf_loc_size */ - /* value_phalf_loc(i) = mpi_rank * 100 + i; */ - /* end do */ + /* Some fake data for this pe to write. */ + for (i = 0; i < pfull_loc_size; i++) + value_pfull_loc[i] = mpi_rank * 100 + i; + for (i = 0; i < phalf_loc_size; i++) + value_phalf_loc[i] = mpi_rank * 100 + i; /* do i = 1, lon_xt_loc_size */ /* do j = 1, lon_yt_loc_size */ /* value_lon_loc(i, j) = mpi_rank * 100 + i +j */ @@ -221,20 +218,18 @@ main(int argc, char **argv) if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - start[0] = pfull_start; - count[0] = pfull_loc_size; - if (nc_put_vara_float(ncid, varid[4], start, count, value_pfull_loc)) ERR; + if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc)) ERR; if (nc_redef(ncid)) ERR; -/* ! Define dimension phalf. */ -/* if (nc_def_dim(ncid, trim(dim_name(4)), dim_len(4), dimid(4))) */ + /* Define dimension phalf. */ + if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; -/* ! Define variable phalf and write data. */ -/* if (nc_def_var(ncid, trim(var_name(6)), var_type(6), dimids=(/dimid(4)/), varid=varid(6))) */ -/* if (nc_var_par_access(ncid, varid(6), NC_INDEPENDENT)) */ -/* if (nc_enddef(ncid)) */ -/* if (nc_put_var(ncid, varid(6), start=(/phalf_start/), count=(/phalf_loc_size/), values=value_phalf_loc)) */ -/* if (nc_redef(ncid)) */ + /* Define variable phalf and write data. */ + if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; + if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_redef(ncid)) ERR; /* ! Define dimension time. */ /* if (nc_def_dim(ncid, trim(dim_name(5)), dim_len(5), dimid(5))) */ @@ -244,7 +239,7 @@ main(int argc, char **argv) /* if (nc_var_par_access(ncid, varid(7), NC_INDEPENDENT)) */ /* if (nc_enddef(ncid)) */ /* ! In NOAA code, do all processors write the single time value? */ -/* if (mpi_rank .eq. 0) then */ +/* if (mpi_rank == 0) then */ /* if (nc_put_var(ncid, varid(7), values=value_time)) */ /* endif */ /* if (nc_redef(ncid)) */ From a645b5912f425591f670b2f2b939967266133509 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 07:39:15 -0600 Subject: [PATCH 08/50] further test development --- nc_test4/tst_gfs_data_1.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 009e5a5e15..55259ce50d 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -51,7 +51,8 @@ main(int argc, char **argv) int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; /* integer :: var_ndims(NUM_VARS) = (/ 1, 2, 1, 2, 1, 1, 1, 4 /) */ /* integer :: ideflate = 4 */ - /* real*8 :: value_time = 2.0, value_time_in */ + float value_time = 2.0; + /* float value_time_in; */ /* real, allocatable :: value_clwmr(:,:,:,:) */ size_t pfull_loc_size, pfull_start; float *value_pfull_loc, *value_pfull_loc_in; @@ -231,18 +232,18 @@ main(int argc, char **argv) if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; if (nc_redef(ncid)) ERR; -/* ! Define dimension time. */ -/* if (nc_def_dim(ncid, trim(dim_name(5)), dim_len(5), dimid(5))) */ + /* Define dimension time. */ + if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; -/* ! Define variable time and write data. */ -/* if (nc_def_var(ncid, trim(var_name(7)), var_type(7), dimids=(/dimid(5)/), varid=varid(7))) */ -/* if (nc_var_par_access(ncid, varid(7), NC_INDEPENDENT)) */ -/* if (nc_enddef(ncid)) */ -/* ! In NOAA code, do all processors write the single time value? */ -/* if (mpi_rank == 0) then */ -/* if (nc_put_var(ncid, varid(7), values=value_time)) */ -/* endif */ -/* if (nc_redef(ncid)) */ + /* Define variable time and write data. */ + if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; + if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)); + if (nc_enddef(ncid)) ERR; + + /* In NOAA code, do all processors write the single time value? */ + if (mpi_rank == 0) + if (nc_put_var_float(ncid, varid[6], &value_time)); + if (nc_redef(ncid)); /* ! Write variable grid_xt data. */ /* if (nc_enddef(ncid)) */ From 306628e9e4910d9b5ee3d1e6cec0056ee072aa51 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 08:12:31 -0600 Subject: [PATCH 09/50] more test development --- nc_test4/tst_gfs_data_1.c | 181 ++++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 86 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 55259ce50d..8b242af288 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -37,7 +37,7 @@ main(int argc, char **argv) MPI_Info info = MPI_INFO_NULL; int ncid; - /* size_t start[NDIM4], count[NDIM4]; */ + size_t start[NDIM4], count[NDIM4]; /* Dimensions. */ char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -51,25 +51,25 @@ main(int argc, char **argv) int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; /* integer :: var_ndims(NUM_VARS) = (/ 1, 2, 1, 2, 1, 1, 1, 4 /) */ /* integer :: ideflate = 4 */ - float value_time = 2.0; + double value_time = 2.0; /* float value_time_in; */ /* real, allocatable :: value_clwmr(:,:,:,:) */ size_t pfull_loc_size, pfull_start; float *value_pfull_loc, *value_pfull_loc_in; size_t phalf_loc_size, phalf_start; float *value_phalf_loc, *value_phalf_loc_in; - /* integer :: grid_xt_loc_size, grid_xt_start */ - /* real, allocatable :: value_grid_xt_loc(:), value_grid_xt_loc_in(:) */ - /* integer :: grid_yt_loc_size, grid_yt_start */ - /* real, allocatable :: value_grid_yt_loc(:), value_grid_yt_loc_in(:) */ - /* integer :: lon_xt_loc_size, lon_xt_start, lon_yt_loc_size, lon_yt_start */ - /* real, allocatable :: value_lon_loc(:,:), value_lon_loc_in(:,:) */ - /* integer :: lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start */ - /* real, allocatable :: value_lat_loc(:,:), value_lat_loc_in(:,:) */ - /* real, allocatable :: value_clwmr_loc(:,:,:,:), value_clwmr_loc_in(:,:,:,:) */ + size_t grid_xt_loc_size, grid_xt_start; + double *value_grid_xt_loc, *value_grid_xt_loc_in; + size_t grid_yt_loc_size, grid_yt_start; + double *value_grid_yt_loc, *value_grid_yt_loc_in; + size_t lon_xt_loc_size, lon_xt_start, lon_yt_loc_size, lon_yt_start; + double *value_lon_loc, *value_lon_loc_in; + size_t lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start; + double *value_lat_loc, *value_lat_loc_in; + float *value_clwmr_loc, *value_clwmr_loc_in; int f; - int i; + int i, j, k; /* int res; */ /* int *slab_data; /\* one slab *\/ */ @@ -78,20 +78,18 @@ main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - /* ! Size of local (i.e. for this pe) grid_xt data. */ - /* grid_xt_loc_size = dim_len(1)/npes; */ - /* grid_xt_start = mpi_rank * grid_xt_loc_size + 1 */ - /* if (mpi_rank .eq. npes - 1) then */ - /* grid_xt_loc_size = grid_xt_loc_size + mod(dim_len(1), npes) */ - /* endif */ + /* Size of local (i.e. for this pe) grid_xt data. */ + grid_xt_loc_size = dim_len[0]/mpi_size; + grid_xt_start = mpi_rank * grid_xt_loc_size; + if (mpi_rank == mpi_size - 1) + grid_xt_loc_size = grid_xt_loc_size + dim_len[0] % mpi_size; /* !print *, mpi_rank, 'grid_xt', dim_len(3), grid_xt_start, grid_xt_loc_size */ - /* ! Size of local (i.e. for this pe) grid_yt data. */ - /* grid_yt_loc_size = dim_len(2)/npes; */ - /* grid_yt_start = mpi_rank * grid_yt_loc_size + 1 */ - /* if (mpi_rank .eq. npes - 1) then */ - /* grid_yt_loc_size = grid_yt_loc_size + mod(dim_len(2), npes) */ - /* endif */ + /* Size of local (i.e. for this pe) grid_yt data. */ + grid_yt_loc_size = dim_len[1]/mpi_size; + grid_yt_start = mpi_rank * grid_yt_loc_size; + if (mpi_rank == mpi_size - 1) + grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size; /* !print *, mpi_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ /* Size of local (i.e. for this pe) pfull data. */ @@ -108,59 +106,67 @@ main(int argc, char **argv) phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; /* !print *, mpi_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ - /* ! Size of local arrays (i.e. for this pe) lon and lat data. This is */ - /* ! specific to 4 pes. */ - /* lon_xt_loc_size = 1536 */ - /* lat_xt_loc_size = 1536 */ - /* if (mpi_rank == 0 .or. mpi_rank == 2) then */ - /* lon_xt_start = 1 */ - /* lat_xt_start = 1 */ - /* else */ - /* lon_xt_start = 1537 */ - /* lat_xt_start = 1537 */ - /* endif */ - /* lon_yt_loc_size = 768 */ - /* lat_yt_loc_size = 768 */ - /* if (mpi_rank == 0 .or. mpi_rank == 1) then */ - /* lon_yt_start = 1 */ - /* lat_yt_start = 1 */ - /* else */ - /* lon_yt_start = 769 */ - /* lat_yt_start = 769 */ - /* endif */ + /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ + /* specific to 4 pes. */ + lon_xt_loc_size = 1536; + lat_xt_loc_size = 1536; + if (mpi_rank == 0 || mpi_rank == 2) + { + lon_xt_start = 1; + lat_xt_start = 1; + } + else + { + lon_xt_start = 1537; + lat_xt_start = 1537; + } + lon_yt_loc_size = 768; + lat_yt_loc_size = 768; + if (mpi_rank == 0 || mpi_rank == 1) + { + lon_yt_start = 1; + lat_yt_start = 1; + } + else + { + lon_yt_start = 769; + lat_yt_start = 769; + } /* ! print *, mpi_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ /* ! print *, mpi_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ /* ! Allocate space on this pe to hold the data for this pe. */ - /* allocate(value_grid_xt_loc(grid_xt_loc_size)) */ - /* allocate(value_grid_xt_loc_in(grid_xt_loc_size)) */ - /* allocate(value_grid_yt_loc(grid_yt_loc_size)) */ - /* allocate(value_grid_yt_loc_in(grid_yt_loc_size)) */ if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; if (!(value_pfull_loc_in = malloc(pfull_loc_size * sizeof(float)))) ERR; if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; if (!(value_phalf_loc_in = malloc(phalf_loc_size * sizeof(float)))) ERR; - /* allocate(value_lon_loc(lon_xt_loc_size, lon_yt_loc_size)) */ - /* allocate(value_lon_loc_in(lon_xt_loc_size, lon_yt_loc_size)) */ - /* allocate(value_lat_loc(lat_xt_loc_size, lat_yt_loc_size)) */ - /* allocate(value_lat_loc_in(lat_xt_loc_size, lat_yt_loc_size)) */ - /* allocate(value_clwmr_loc(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ - /* allocate(value_clwmr_loc_in(lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1)) */ + if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; + if (!(value_grid_xt_loc_in = malloc(grid_xt_loc_size * sizeof(double)))) ERR; + if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; + if (!(value_grid_yt_loc_in = malloc(grid_yt_loc_size * sizeof(double)))) ERR; + if (!(value_lon_loc = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; + if (!(value_lon_loc_in = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; + if (!(value_lat_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; + if (!(value_lat_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; + if (!(value_clwmr_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; + if (!(value_clwmr_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; /* Some fake data for this pe to write. */ for (i = 0; i < pfull_loc_size; i++) value_pfull_loc[i] = mpi_rank * 100 + i; for (i = 0; i < phalf_loc_size; i++) value_phalf_loc[i] = mpi_rank * 100 + i; - /* do i = 1, lon_xt_loc_size */ - /* do j = 1, lon_yt_loc_size */ - /* value_lon_loc(i, j) = mpi_rank * 100 + i +j */ - /* value_lat_loc(i, j) = mpi_rank * 100 + i +j */ - /* do k = 1, pfull_loc_size */ - /* value_clwmr_loc(i, j, k, 1) = mpi_rank * 100 + i + j + k */ - /* end do */ - /* end do */ - /* end do */ + for (j = 0; j < lon_yt_loc_size; j++) + { + for(i = 0; i < lon_xt_loc_size; i++) + { + value_lon_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j; + value_lat_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j; + for (k = 0; k < pfull_loc_size; k++) + value_clwmr_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j + k; + } + } + /* Allocate data. */ /* if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ @@ -237,24 +243,27 @@ main(int argc, char **argv) /* Define variable time and write data. */ if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; - if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)); + if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; /* In NOAA code, do all processors write the single time value? */ if (mpi_rank == 0) - if (nc_put_var_float(ncid, varid[6], &value_time)); + if (nc_put_var_double(ncid, varid[6], &value_time)); if (nc_redef(ncid)); -/* ! Write variable grid_xt data. */ -/* if (nc_enddef(ncid)) */ -/* if (nc_put_var(ncid, varid(1), start=(/grid_xt_start/), count=(/grid_xt_loc_size/), values=value_grid_xt_loc)) */ -/* if (nc_redef(ncid)) */ + /* Write variable grid_xt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_redef(ncid)) ERR; -/* ! Write lon data. */ -/* if (nc_enddef(ncid)) */ -/* if (nc_put_var(ncid, varid(2), start=(/lon_xt_start, lon_yt_start/), count=(/lon_xt_loc_size, lon_yt_loc_size/), & */ -/* values=value_lon_loc)) */ -/* if (nc_redef(ncid)) */ + /* Write lon data. */ + if (nc_enddef(ncid)) ERR; + start[0] = lon_xt_start; + start[1] = lon_yt_start; + count[0] = lon_xt_loc_size; + count[1] = lon_yt_loc_size; + /* if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; */ + if (nc_redef(ncid)) ERR; /* ! Write grid_yt data. */ /* if (nc_enddef(ncid)) */ @@ -375,20 +384,20 @@ main(int argc, char **argv) /* free(slab_data); */ /* Free resources. */ - /* free(value_grid_xt_loc) */ - /* free(value_grid_xt_loc_in) */ - /* free(value_grid_yt_loc) */ - /* free(value_grid_yt_loc_in) */ + free(value_grid_xt_loc); + free(value_grid_xt_loc_in); + free(value_grid_yt_loc); + free(value_grid_yt_loc_in); free(value_pfull_loc); free(value_pfull_loc_in); - /* free(value_phalf_loc) */ - /* free(value_phalf_loc_in) */ - /* free(value_lon_loc) */ - /* free(value_lon_loc_in) */ - /* free(value_lat_loc) */ - /* free(value_lat_loc_in) */ - /* free(value_clwmr_loc) */ - /* free(value_clwmr_loc_in) */ + free(value_phalf_loc); + free(value_phalf_loc_in); + free(value_lon_loc); + free(value_lon_loc_in); + free(value_lat_loc); + free(value_lat_loc_in); + free(value_clwmr_loc); + free(value_clwmr_loc_in); } /* Shut down MPI. */ From 467f342ae92bd8f2268fc5d195d8b449fa891552 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 08:35:11 -0600 Subject: [PATCH 10/50] further test development --- libhdf5/hdf5var.c | 1 + nc_test4/tst_gfs_data_1.c | 65 ++++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index ce0b234652..5cdaebc130 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1532,6 +1532,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : var->dim[i]->len; stride[i] = stridep ? stridep[i] : 1; + LOG((3, "%s: start[%d] %ld count[%d] %ld stride[%d] %ld", __func__, i, start[i], i, count[i], i, stride[i])); /* Check to see if any counts are zero. */ if (!count[i]) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 8b242af288..019fcc9def 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -43,6 +43,7 @@ main(int argc, char **argv) char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; int dimid[NDIM5]; + int dimid_data[NDIM4]; /* Variables. */ #define NUM_VARS 8 @@ -112,25 +113,25 @@ main(int argc, char **argv) lat_xt_loc_size = 1536; if (mpi_rank == 0 || mpi_rank == 2) { - lon_xt_start = 1; - lat_xt_start = 1; + lon_xt_start = 0; + lat_xt_start = 0; } else { - lon_xt_start = 1537; - lat_xt_start = 1537; + lon_xt_start = 1536; + lat_xt_start = 1536; } lon_yt_loc_size = 768; lat_yt_loc_size = 768; if (mpi_rank == 0 || mpi_rank == 1) { - lon_yt_start = 1; - lat_yt_start = 1; + lon_yt_start = 0; + lat_yt_start = 0; } else { - lon_yt_start = 769; - lat_yt_start = 769; + lon_yt_start = 768; + lat_yt_start = 768; } /* ! print *, mpi_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ /* ! print *, mpi_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ @@ -167,14 +168,6 @@ main(int argc, char **argv) } } - /* Allocate data. */ - /* if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ - - /* Create phony data. We're going to write a 24x24 array of ints, - in 4 sets of 144. */ - /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ - /* slab_data[i] = mpi_rank; */ - if (!mpi_rank) printf("\n*** Testing parallel writes with compression filters.\n"); { @@ -262,26 +255,34 @@ main(int argc, char **argv) start[1] = lon_yt_start; count[0] = lon_xt_loc_size; count[1] = lon_yt_loc_size; - /* if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; */ + nc_set_log_level(3); + if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; if (nc_redef(ncid)) ERR; -/* ! Write grid_yt data. */ -/* if (nc_enddef(ncid)) */ -/* if (nc_put_var(ncid, varid(3), start=(/grid_yt_start/), count=(/grid_yt_loc_size/), values=value_grid_yt_loc)) */ -/* if (nc_redef(ncid)) */ + /* Write grid_yt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_redef(ncid)) ERR; -/* ! Write lat data. */ -/* if (nc_enddef(ncid)) */ -/* if (nc_put_var(ncid, varid(4), start=(/lat_xt_start, lat_yt_start/), count=(/lat_xt_loc_size, lat_yt_loc_size/), & */ -/* values=value_lat_loc)) */ -/* if (nc_redef(ncid)) */ + /* Write lat data. */ + if (nc_enddef(ncid)) ERR; + start[0] = lat_xt_start; + start[1] = lat_yt_start; + count[0] = lat_xt_loc_size; + count[1] = lat_yt_loc_size; + if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; + if (nc_redef(ncid)) ERR; -/* ! Define variable clwmr and write data (?) */ -/* if (nc_def_var(ncid, trim(var_name(8)), var_type(8), dimids=(/dimid(1), dimid(2), dimid(3), dimid(5)/), & */ -/* varid=varid(8), shuffle=.true., deflate_level=ideflate)) */ -/* if (nc_var_par_access(ncid, varid(8), NC_COLLECTIVE)) */ -/* if (nc_enddef(ncid)) */ -/* ! if (nc_put_var(ncid, varid(8), values=value_clwmr)) */ + + /* Define variable clwmr and write data (?) */ + dimid_data[0] = dimid[4]; + dimid_data[1] = dimid[2]; + dimid_data[2] = dimid[1]; + dimid_data[3] = dimid[0]; + if (nc_def_var(ncid, var_name[7], var_type[7], NDIM4, dimid_data, &varid[7])) ERR; + /* , shuffle=.true., deflate_level=ideflate */ + if (nc_var_par_access(ncid, varid[7], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; /* if (nc_put_var(ncid, varid(8), start=(/lat_xt_start, lat_yt_start, pfull_start, 1/), & */ /* count=(/lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1/), values=value_clwmr_loc)) */ /* if (nc_redef(ncid)) */ From dc37446a5fd2e6bad239fb50468e11773c5b90b0 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:01:24 -0600 Subject: [PATCH 11/50] more test development --- libhdf5/hdf5var.c | 2 +- nc_test4/tst_gfs_data_1.c | 39 ++++++++++++--------------------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 5cdaebc130..0e4f74dbee 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1532,7 +1532,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : var->dim[i]->len; stride[i] = stridep ? stridep[i] : 1; - LOG((3, "%s: start[%d] %ld count[%d] %ld stride[%d] %ld", __func__, i, start[i], i, count[i], i, stride[i])); + LOG((4, "start[%d] %ld count[%d] %ld stride[%d] %ld", i, start[i], i, count[i], i, stride[i])); /* Check to see if any counts are zero. */ if (!count[i]) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 019fcc9def..301684050c 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -283,12 +283,19 @@ main(int argc, char **argv) /* , shuffle=.true., deflate_level=ideflate */ if (nc_var_par_access(ncid, varid[7], NC_COLLECTIVE)) ERR; if (nc_enddef(ncid)) ERR; -/* if (nc_put_var(ncid, varid(8), start=(/lat_xt_start, lat_yt_start, pfull_start, 1/), & */ -/* count=(/lat_xt_loc_size, lat_yt_loc_size, pfull_loc_size, 1/), values=value_clwmr_loc)) */ -/* if (nc_redef(ncid)) */ + start[0] = 0; + start[1] = pfull_start; + start[2] = lat_yt_start; + start[3] = lat_xt_start; + count[0] = 1; + count[1] = pfull_loc_size; + count[2] = lat_yt_loc_size; + count[3] = lat_xt_loc_size; + if (nc_put_vara_float(ncid, varid[7], start, count, value_clwmr_loc)) ERR; + if (nc_redef(ncid)) ERR; -/* ! Close the file. */ -/* if (nc_close(ncid)) */ + /* Close the file. */ + if (nc_close(ncid)) ERR; /* Setting any filter only will work for HDF5-1.10.3 and later * versions. */ @@ -315,28 +322,6 @@ main(int argc, char **argv) /* if (res != NC_EINVAL) ERR; */ /* #endif */ - /* /\* Write metadata to file. *\/ */ - /* if (nc_enddef(ncid)) ERR; */ - - /* /\* Set up slab for this process. *\/ */ - /* start[0] = mpi_rank * DIMSIZE/mpi_size; */ - /* start[1] = 0; */ - /* count[0] = DIMSIZE/mpi_size; */ - /* count[1] = DIMSIZE; */ - /* count[2] = 1; */ - /* /\*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n", */ - /* mpi_rank, start[0], start[1], count[0], count[1]);*\/ */ - - /* /\* Should not be allowed to change access to independent, */ - /* * because filters are in use. *\/ */ - /* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR; */ - - /* /\* Write slabs of data. *\/ */ - /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ - /* if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR; */ - - /* Close the netcdf file. */ - if (nc_close(ncid)) ERR; /* /\* Check file. *\/ */ /* { */ From 5ce5f1cd1fcaa76fc3f6345546a3781512698917 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:05:56 -0600 Subject: [PATCH 12/50] more test development --- nc_test4/tst_gfs_data_1.c | 47 +++++++++++++++------------------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 301684050c..0ff43c6941 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -71,7 +71,7 @@ main(int argc, char **argv) int f; int i, j, k; - /* int res; */ + int res; /* int *slab_data; /\* one slab *\/ */ /* Initialize MPI. */ @@ -255,7 +255,6 @@ main(int argc, char **argv) start[1] = lon_yt_start; count[0] = lon_xt_loc_size; count[1] = lon_yt_loc_size; - nc_set_log_level(3); if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; if (nc_redef(ncid)) ERR; @@ -280,7 +279,23 @@ main(int argc, char **argv) dimid_data[2] = dimid[1]; dimid_data[3] = dimid[0]; if (nc_def_var(ncid, var_name[7], var_type[7], NDIM4, dimid_data, &varid[7])) ERR; - /* , shuffle=.true., deflate_level=ideflate */ + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, varid[7], s, 1, 4); + else + { + res = nc_def_var_deflate(ncid, varid[7], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, varid[7], 32, 32); + } +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + if (nc_var_par_access(ncid, varid[7], NC_COLLECTIVE)) ERR; if (nc_enddef(ncid)) ERR; start[0] = 0; @@ -297,32 +312,6 @@ main(int argc, char **argv) /* Close the file. */ if (nc_close(ncid)) ERR; - /* Setting any filter only will work for HDF5-1.10.3 and later - * versions. */ -/* if (!f) */ -/* res = nc_def_var_deflate(ncid, 0, s, 1, 1); */ -/* else */ -/* { */ -/* res = nc_def_var_deflate(ncid, 0, s, 0, 0); */ -/* if (!res) */ -/* res = nc_def_var_szip(ncid, 0, 32, 32); */ -/* } */ -/* #ifdef HDF5_SUPPORTS_PAR_FILTERS */ -/* if (res) ERR; */ -/* #else */ -/* if (res != NC_EINVAL) ERR; */ -/* #endif */ - -/* /\* Setting fletcher32 only will work for HDF5-1.10.3 and later */ -/* * versions. *\/ */ -/* res = nc_def_var_fletcher32(ncid, 0, 1); */ -/* #ifdef HDF5_SUPPORTS_PAR_FILTERS */ -/* if (res) ERR; */ -/* #else */ -/* if (res != NC_EINVAL) ERR; */ -/* #endif */ - - /* /\* Check file. *\/ */ /* { */ /* int shuffle_in, deflate_in, deflate_level_in; */ From 179b258319d8422d5a6ada42620188e307235abc Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:18:14 -0600 Subject: [PATCH 13/50] more test development --- nc_test4/tst_gfs_data_1.c | 97 +++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 44 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 0ff43c6941..0f449e0187 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -14,11 +14,10 @@ #include #define FILE_NAME "tst_gfs_data_1.nc" +#define NUM_VARS 8 #define NDIM4 4 #define NDIM5 5 -#define QTR_DATA (DIMSIZE * DIMSIZE / 4) #define NUM_PROC 4 -#define NUM_SLABS 10 #define NUM_SHUFFLE_SETTINGS 1 /* #define NUM_SHUFFLE_SETTINGS 2 */ #ifdef HAVE_H5Z_SZIP @@ -46,11 +45,10 @@ main(int argc, char **argv) int dimid_data[NDIM4]; /* Variables. */ -#define NUM_VARS 8 char var_name[NUM_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", "lat", "pfull", "phalf", "time", "clwmr"}; int varid[NUM_VARS]; int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; - /* integer :: var_ndims(NUM_VARS) = (/ 1, 2, 1, 2, 1, 1, 1, 4 /) */ + int var_ndims[NUM_VARS] = {1, 2, 1, 2, 1, 1, 1, 4}; /* integer :: ideflate = 4 */ double value_time = 2.0; /* float value_time_in; */ @@ -72,7 +70,6 @@ main(int argc, char **argv) int f; int i, j, k; int res; - /* int *slab_data; /\* one slab *\/ */ /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -312,45 +309,57 @@ main(int argc, char **argv) /* Close the file. */ if (nc_close(ncid)) ERR; - /* /\* Check file. *\/ */ - /* { */ - /* int shuffle_in, deflate_in, deflate_level_in; */ - /* int options_mask_in, pixels_per_block_in; */ - /* int *slab_data_in; */ - - /* /\* Allocate data. *\/ */ - /* if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; */ - - /* /\* Reopen the file for parallel access. *\/ */ - /* if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; */ - - /* /\* Check state of compression. *\/ */ - /* if (!f) */ - /* { */ - /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; */ - /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ - /* if (!deflate_in || deflate_level_in != 1) ERR; */ - /* } */ - /* else */ - /* { */ - /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; */ - /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ - /* if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; */ - /* } */ - - /* /\* Use parallel I/O to read the data. *\/ */ - /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ - /* { */ - /* if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; */ - /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ - /* if (slab_data_in[i] != mpi_rank) ERR; */ - /* } */ - - /* /\* Close the netcdf file. *\/ */ - /* if (nc_close(ncid)) ERR; */ - - /* free(slab_data_in); */ - /* } */ + /* Check file. */ + { + int ndims, nvars, natts, unlimdimid; + int v; + + /* int shuffle_in, deflate_in, deflate_level_in; */ + /* int options_mask_in, pixels_per_block_in; */ + + /* Reopen the file for parallel access. */ + if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; + + /* Check file. */ + if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; + if (ndims != NDIM5 || nvars != NUM_VARS || natts != 0 || unlimdimid != -1) ERR; + + /* Check vars. */ + for (v = 0; v < NUM_VARS; v++) + { + char name_in[NC_MAX_NAME + 1]; + int xtype_in, ndims_in, natts_in; + int dimids_in[NDIM4]; + + if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; + if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v] || ndims_in != var_ndims[v]) ERR; + } + + /* /\* Check state of compression. *\/ */ + /* if (!f) */ + /* { */ + /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; */ + /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ + /* if (!deflate_in || deflate_level_in != 1) ERR; */ + /* } */ + /* else */ + /* { */ + /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; */ + /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ + /* if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; */ + /* } */ + + /* /\* Use parallel I/O to read the data. *\/ */ + /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ + /* { */ + /* if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; */ + /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ + /* if (slab_data_in[i] != mpi_rank) ERR; */ + /* } */ + + /* Close the netcdf file. */ + if (nc_close(ncid)) ERR; + } if (!mpi_rank) SUMMARIZE_ERR; From 0672662f31edd65b718d1442ef3663f3a69698d9 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:31:52 -0600 Subject: [PATCH 14/50] now reopening file and checking dim and var metadata --- nc_test4/tst_gfs_data_1.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 0f449e0187..b9c75f0780 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -312,7 +312,8 @@ main(int argc, char **argv) /* Check file. */ { int ndims, nvars, natts, unlimdimid; - int v; + char name_in[NC_MAX_NAME + 1]; + int d, v; /* int shuffle_in, deflate_in, deflate_level_in; */ /* int options_mask_in, pixels_per_block_in; */ @@ -324,10 +325,18 @@ main(int argc, char **argv) if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; if (ndims != NDIM5 || nvars != NUM_VARS || natts != 0 || unlimdimid != -1) ERR; + /* Check dims. */ + for (d = 0; d < NDIM5; d++) + { + size_t len_in; + + if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR; + if (strcmp(name_in, dim_name[d]) && len_in != dim_len[d]) ERR; + } + /* Check vars. */ for (v = 0; v < NUM_VARS; v++) { - char name_in[NC_MAX_NAME + 1]; int xtype_in, ndims_in, natts_in; int dimids_in[NDIM4]; From 823240d17a6c7d4c87e8fcbc9cf8bf25a7860af1 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:35:08 -0600 Subject: [PATCH 15/50] now checking pfull data --- nc_test4/tst_gfs_data_1.c | 41 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index b9c75f0780..bf9ebf25b7 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -1,12 +1,12 @@ /* - Copyright 2020, UCAR/Unidata See COPYRIGHT file for copying and - redistribution conditions. + Copyright 2020, UCAR/Unidata See COPYRIGHT file for copying and + redistribution conditions. - This program tests netcdf-4 parallel I/O using the same access - pattern as is used by NOAA's GFS when writing and reading model - data. + This program tests netcdf-4 parallel I/O using the same access + pattern as is used by NOAA's GFS when writing and reading model + data. - Ed Hartnett, 6/28/20 + Ed Hartnett, 6/28/20 */ #include @@ -86,17 +86,17 @@ main(int argc, char **argv) /* Size of local (i.e. for this pe) grid_yt data. */ grid_yt_loc_size = dim_len[1]/mpi_size; grid_yt_start = mpi_rank * grid_yt_loc_size; - if (mpi_rank == mpi_size - 1) + if (mpi_rank == mpi_size - 1) grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size; /* !print *, mpi_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ /* Size of local (i.e. for this pe) pfull data. */ pfull_loc_size = dim_len[2]/mpi_size; pfull_start = mpi_rank * pfull_loc_size; - if (mpi_rank == mpi_size - 1) + if (mpi_rank == mpi_size - 1) pfull_loc_size = pfull_loc_size + dim_len[2] % mpi_size; /* !print *, mpi_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ - + /* Size of local (i.e. for this pe) phalf data. */ phalf_loc_size = dim_len[3]/mpi_size; phalf_start = mpi_rank * phalf_loc_size; @@ -148,7 +148,7 @@ main(int argc, char **argv) if (!(value_lat_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; if (!(value_clwmr_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; if (!(value_clwmr_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; - + /* Some fake data for this pe to write. */ for (i = 0; i < pfull_loc_size; i++) value_pfull_loc[i] = mpi_rank * 100 + i; @@ -164,9 +164,9 @@ main(int argc, char **argv) value_clwmr_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j + k; } } - + if (!mpi_rank) - printf("\n*** Testing parallel writes with compression filters.\n"); + printf("\n*** Testing parallel writes with compression filters.\n"); { int s; for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) @@ -182,7 +182,7 @@ main(int argc, char **argv) /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - + /* Turn off fill mode. */ if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; @@ -235,7 +235,7 @@ main(int argc, char **argv) if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - + /* In NOAA code, do all processors write the single time value? */ if (mpi_rank == 0) if (nc_put_var_double(ncid, varid[6], &value_time)); @@ -269,7 +269,7 @@ main(int argc, char **argv) if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; if (nc_redef(ncid)) ERR; - + /* Define variable clwmr and write data (?) */ dimid_data[0] = dimid[4]; dimid_data[1] = dimid[2]; @@ -329,7 +329,7 @@ main(int argc, char **argv) for (d = 0; d < NDIM5; d++) { size_t len_in; - + if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR; if (strcmp(name_in, dim_name[d]) && len_in != dim_len[d]) ERR; } @@ -339,11 +339,16 @@ main(int argc, char **argv) { int xtype_in, ndims_in, natts_in; int dimids_in[NDIM4]; - + if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v] || ndims_in != var_ndims[v]) ERR; } + /* Check pfull data. */ + if (nc_get_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc_in)) ERR; + for (i = 0; i < pfull_loc_size; i++) + if (value_pfull_loc_in[i] != value_pfull_loc[i]) ERR; + /* /\* Check state of compression. *\/ */ /* if (!f) */ /* { */ @@ -397,7 +402,7 @@ main(int argc, char **argv) MPI_Finalize(); if (!mpi_rank) - FINAL_RESULTS; + FINAL_RESULTS; return 0; } From 43c5f08dbb7ef86249872de1e82930687987abda Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:39:38 -0600 Subject: [PATCH 16/50] now checking more data --- nc_test4/tst_gfs_data_1.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index bf9ebf25b7..6dd58a83e7 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -238,8 +238,8 @@ main(int argc, char **argv) /* In NOAA code, do all processors write the single time value? */ if (mpi_rank == 0) - if (nc_put_var_double(ncid, varid[6], &value_time)); - if (nc_redef(ncid)); + if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; + if (nc_redef(ncid)) ERR; /* Write variable grid_xt data. */ if (nc_enddef(ncid)) ERR; @@ -313,6 +313,7 @@ main(int argc, char **argv) { int ndims, nvars, natts, unlimdimid; char name_in[NC_MAX_NAME + 1]; + double value_time_in; int d, v; /* int shuffle_in, deflate_in, deflate_level_in; */ @@ -349,6 +350,17 @@ main(int argc, char **argv) for (i = 0; i < pfull_loc_size; i++) if (value_pfull_loc_in[i] != value_pfull_loc[i]) ERR; + /* Check phalf data. */ + if (nc_get_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc_in)) ERR; + for (i = 0; i < phalf_loc_size; i++) + if (value_phalf_loc_in[i] != value_phalf_loc[i]) ERR; + + if (mpi_rank == 0) + { + if (nc_get_var_double(ncid, varid[6], &value_time_in)) ERR; + if (value_time_in != value_time) ERR; + } + /* /\* Check state of compression. *\/ */ /* if (!f) */ /* { */ From 1d06c30cf7e53681bfced27bb482e89e20971a40 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:41:22 -0600 Subject: [PATCH 17/50] now checking more data --- nc_test4/tst_gfs_data_1.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index 6dd58a83e7..a3b1f3852b 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -354,13 +354,20 @@ main(int argc, char **argv) if (nc_get_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc_in)) ERR; for (i = 0; i < phalf_loc_size; i++) if (value_phalf_loc_in[i] != value_phalf_loc[i]) ERR; - + + /* Check time. */ if (mpi_rank == 0) { if (nc_get_var_double(ncid, varid[6], &value_time_in)) ERR; if (value_time_in != value_time) ERR; } + /* Check grid_xt data. */ + if (nc_get_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc_in)) ERR; + + /* Check grid_yt data. */ + if (nc_get_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc_in)) ERR; + /* /\* Check state of compression. *\/ */ /* if (!f) */ /* { */ From 8d7d9a598ebdfca9a280fb581bde59ac6bb52933 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jun 2020 09:44:55 -0600 Subject: [PATCH 18/50] now checking more data --- nc_test4/tst_gfs_data_1.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nc_test4/tst_gfs_data_1.c b/nc_test4/tst_gfs_data_1.c index a3b1f3852b..c7c5ae4e6f 100644 --- a/nc_test4/tst_gfs_data_1.c +++ b/nc_test4/tst_gfs_data_1.c @@ -154,6 +154,10 @@ main(int argc, char **argv) value_pfull_loc[i] = mpi_rank * 100 + i; for (i = 0; i < phalf_loc_size; i++) value_phalf_loc[i] = mpi_rank * 100 + i; + for (i = 0; i < grid_xt_loc_size; i++) + value_grid_xt_loc[i] = mpi_rank * 100 + i; + for (i = 0; i < grid_yt_loc_size; i++) + value_grid_yt_loc[i] = mpi_rank * 100 + i; for (j = 0; j < lon_yt_loc_size; j++) { for(i = 0; i < lon_xt_loc_size; i++) @@ -364,9 +368,13 @@ main(int argc, char **argv) /* Check grid_xt data. */ if (nc_get_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc_in)) ERR; + for (i = 0; i < grid_xt_loc_size; i++) + if (value_grid_xt_loc_in[i] != value_grid_xt_loc[i]) ERR; /* Check grid_yt data. */ if (nc_get_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc_in)) ERR; + for (i = 0; i < grid_yt_loc_size; i++) + if (value_grid_yt_loc_in[i] != value_grid_yt_loc[i]) ERR; /* /\* Check state of compression. *\/ */ /* if (!f) */ From ba6ab38a1106db8154762d688df3489c98dd6690 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 30 Jun 2020 10:40:05 -0600 Subject: [PATCH 19/50] fixed warnings in tst_create_files.c --- configure.ac | 1 + nc_perf/Makefile.am | 9 +++++---- nc_perf/run_gfs_test.sh.in | 12 ++++++++++++ nc_perf/run_par_bm_test.sh.in | 2 +- nc_perf/tst_create_files.c | 5 +++-- {nc_test4 => nc_perf}/tst_gfs_data_1.c | 0 nc_test4/Makefile.am | 2 +- nc_test4/run_par_test.sh.in | 3 --- 8 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 nc_perf/run_gfs_test.sh.in rename {nc_test4 => nc_perf}/tst_gfs_data_1.c (100%) diff --git a/configure.ac b/configure.ac index 5b115779f2..d5e937273b 100644 --- a/configure.ac +++ b/configure.ac @@ -1553,6 +1553,7 @@ AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c) AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh]) AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh]) AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh]) +AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh]) AC_CONFIG_FILES([examples/C/run_par_test.sh], [chmod ugo+x examples/C/run_par_test.sh]) AC_CONFIG_FILES([nc-config], [chmod 755 nc-config]) AC_CONFIG_FILES([Makefile diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am index 4c21cb6315..a0a701d3d4 100644 --- a/nc_perf/Makefile.am +++ b/nc_perf/Makefile.am @@ -20,7 +20,8 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la check_PROGRAMS = tst_create_files bm_file tst_chunks3 tst_ar4 \ tst_ar4_3d tst_ar4_4d bm_many_objs tst_h_many_atts bm_many_atts \ tst_files2 tst_files3 tst_mem tst_mem1 tst_knmi bm_netcdf4_recs \ -tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando +tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando \ +tst_gfs_data_1 bm_file_SOURCES = bm_file.c tst_utils.c bm_netcdf4_recs_SOURCES = bm_netcdf4_recs.c tst_utils.c @@ -48,16 +49,16 @@ TESTS += run_bm_test1.sh run_bm_test2.sh run_bm_test1.log: tst_create_files.log run_bm_test2.log: tst_create_files.log -# This will run a parallel I/O benchmark for parallel builds. +# This will run parallel I/O benchmarks for parallel builds. if TEST_PARALLEL4 -TESTS += run_par_bm_test.sh +TESTS += run_par_bm_test.sh run_gfs_test.sh run_par_bm_test.log: tst_create_files.log endif # TEST_PARALLEL4 endif # BUILD_UTILITIES EXTRA_DIST = run_par_bm_test.sh.in run_knmi_bm.sh perftest.sh \ run_bm_test1.sh run_bm_test2.sh run_tst_chunks.sh run_bm_elena.sh \ -CMakeLists.txt +CMakeLists.txt run_gfs_test.sh.in CLEANFILES = tst_*.nc bigmeta.nc bigvars.nc floats*.nc \ floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl tst_*.cdl diff --git a/nc_perf/run_gfs_test.sh.in b/nc_perf/run_gfs_test.sh.in new file mode 100644 index 0000000000..575ad37c9d --- /dev/null +++ b/nc_perf/run_gfs_test.sh.in @@ -0,0 +1,12 @@ +#!/bin/sh + +# This shell file tests the GFS tests parallel I/O. +# Ed Hartnett, 6/30/20 + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +echo "*** Running tst_gfs_1..." +@MPIEXEC@ -n 4 ./tst_gfs_data_1 + +exit 0 diff --git a/nc_perf/run_par_bm_test.sh.in b/nc_perf/run_par_bm_test.sh.in index 6cdf6b1a8d..b51c04308a 100644 --- a/nc_perf/run_par_bm_test.sh.in +++ b/nc_perf/run_par_bm_test.sh.in @@ -1,6 +1,6 @@ #!/bin/sh -# This shell file tests the bm_ile program for parallel I/O. +# This shell file tests the bm_file program for parallel I/O. # Ed Hartnett if test "x$srcdir" = x ; then srcdir=`pwd`; fi diff --git a/nc_perf/tst_create_files.c b/nc_perf/tst_create_files.c index 4a5eebde3e..20f0ca0ba3 100644 --- a/nc_perf/tst_create_files.c +++ b/nc_perf/tst_create_files.c @@ -18,6 +18,7 @@ /* We will create this file. */ #define FILE_NAME "tst_floats_1D.nc" +#define MAX_TYPE_NAME 6 int main(int argc, char **argv) @@ -142,7 +143,7 @@ main(int argc, char **argv) int ncid, dimids[MAX_DIMS], varid; char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1]; - char type_name[MAX_TYPES][NC_MAX_NAME + 1] = {"floats", "ints", "shorts"}; + char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"}; int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT}; size_t len; float fdata[TOTAL_SIZE]; @@ -254,7 +255,7 @@ main(int argc, char **argv) int ncid, dimids[MAX_DIMS], varid; char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1]; - char type_name[MAX_TYPES][NC_MAX_NAME + 1] = {"floats", "ints", "shorts"}; + char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"}; int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT}; size_t len; float fdata[TOTAL_SIZE]; diff --git a/nc_test4/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c similarity index 100% rename from nc_test4/tst_gfs_data_1.c rename to nc_perf/tst_gfs_data_1.c diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 03e455ecd0..82287329c7 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -87,7 +87,7 @@ endif # BUILD_UTILITIES if TEST_PARALLEL4 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \ tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \ -tst_mode tst_parallel_zlib tst_parallel_compress tst_gfs_data_1 +tst_mode tst_parallel_zlib tst_parallel_compress TESTS += run_par_test.sh endif # TEST_PARALLEL4 diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in index dfac5a98b8..fc382bff2e 100644 --- a/nc_test4/run_par_test.sh.in +++ b/nc_test4/run_par_test.sh.in @@ -60,7 +60,4 @@ if test "@HAS_PAR_FILTERS@" = "yes"; then @MPIEXEC@ -n 1 ./tst_parallel_compress @MPIEXEC@ -n 4 ./tst_parallel_compress - echo - echo "Parallel I/O like the NOAA GFS writes and reads data." - @MPIEXEC@ -n 4 ./tst_gfs_data_1 fi From 16a7b34c61b7494e58e72099d7871885b61be4cf Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 30 Jun 2020 10:42:28 -0600 Subject: [PATCH 20/50] fixed warnings in bm_file.c --- nc_perf/bm_file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nc_perf/bm_file.c b/nc_perf/bm_file.c index b758d6d6c2..ee04cfdd7a 100644 --- a/nc_perf/bm_file.c +++ b/nc_perf/bm_file.c @@ -817,7 +817,7 @@ main(int argc, char **argv) extern int opterr; extern char *optarg; char file_in[NC_MAX_NAME + 1], file_out[NC_MAX_NAME + 1] = {""}; - char file_out_2[NC_MAX_NAME + 1]; + char file_out_2[NC_MAX_NAME + 10 + 1]; /* extra 10 to silence warning */ int out_format, in_format, header = 0, doublecheck = 0; int convert_unlim = 0; char *str1, *str2, *token, *subtoken; @@ -1068,7 +1068,7 @@ main(int argc, char **argv) if (doublecheck) { /* We need a string long enough for the copy command. */ - char cmd[NC_MAX_NAME * 2 + 5]; + char cmd[NC_MAX_NAME * 3 + 5]; #ifdef USE_PARALLEL MPI_Barrier(MPI_COMM_WORLD); From 8029db1d4f18543fdec6ddefb7d458575d76beb3 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 30 Jun 2020 13:20:28 -0600 Subject: [PATCH 21/50] further benchmark development, moved to benchmark dir --- nc_perf/Makefile.am | 16 ++++--- nc_perf/tst_gfs_data_1.c | 99 +++++++++++++++++++++++----------------- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am index a0a701d3d4..898cca8b02 100644 --- a/nc_perf/Makefile.am +++ b/nc_perf/Makefile.am @@ -52,18 +52,20 @@ run_bm_test2.log: tst_create_files.log # This will run parallel I/O benchmarks for parallel builds. if TEST_PARALLEL4 TESTS += run_par_bm_test.sh run_gfs_test.sh -run_par_bm_test.log: tst_create_files.log +run_par_bm_test.log: tst_create_files.log run_bm_test1.log endif # TEST_PARALLEL4 endif # BUILD_UTILITIES -EXTRA_DIST = run_par_bm_test.sh.in run_knmi_bm.sh perftest.sh \ -run_bm_test1.sh run_bm_test2.sh run_tst_chunks.sh run_bm_elena.sh \ -CMakeLists.txt run_gfs_test.sh.in +# Extra files for the dist. Note that parallel tests end in .in, +# because configure substitute in the launcher (usually mpiexec). +EXTRA_DIST = run_knmi_bm.sh perftest.sh run_bm_test1.sh \ +run_bm_test2.sh run_tst_chunks.sh run_bm_elena.sh CMakeLists.txt \ +run_gfs_test.sh.in run_par_bm_test.sh.in -CLEANFILES = tst_*.nc bigmeta.nc bigvars.nc floats*.nc \ -floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl tst_*.cdl +CLEANFILES = tst_*.nc bigmeta.nc bigvars.nc floats*.nc floats*.cdl \ +shorts*.nc shorts*.cdl ints*.nc ints*.cdl tst_*.cdl -DISTCLEANFILES = run_par_bm_test.sh MSGCPP_CWP_NC*.nc +DISTCLEANFILES = run_par_bm_test.sh MSGCPP_CWP_NC*.nc run_gfs_test.sh # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index c7c5ae4e6f..61cbd8165c 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -10,6 +10,8 @@ */ #include +#include +#include /* Extra high precision time info. */ #include "err_macros.h" #include @@ -26,28 +28,37 @@ #else #define NUM_COMPRESSION_FILTERS 1 #endif +#define MILLION 1000000 int main(int argc, char **argv) { /* MPI stuff. */ - int mpi_size, mpi_rank; + int mpi_size, my_rank; MPI_Comm comm = MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; + /* For timing. */ + double ftime; + int write_us; + int ncid; size_t start[NDIM4], count[NDIM4]; /* Dimensions. */ - char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; + char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", + "phalf", "time"}; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; int dimid[NDIM5]; int dimid_data[NDIM4]; /* Variables. */ - char var_name[NUM_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", "lat", "pfull", "phalf", "time", "clwmr"}; + char var_name[NUM_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", + "lat", "pfull", "phalf", "time", + "clwmr"}; int varid[NUM_VARS]; - int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; + int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, + NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; int var_ndims[NUM_VARS] = {1, 2, 1, 2, 1, 1, 1, 4}; /* integer :: ideflate = 4 */ double value_time = 2.0; @@ -74,41 +85,41 @@ main(int argc, char **argv) /* Initialize MPI. */ MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /* Size of local (i.e. for this pe) grid_xt data. */ grid_xt_loc_size = dim_len[0]/mpi_size; - grid_xt_start = mpi_rank * grid_xt_loc_size; - if (mpi_rank == mpi_size - 1) + grid_xt_start = my_rank * grid_xt_loc_size; + if (my_rank == mpi_size - 1) grid_xt_loc_size = grid_xt_loc_size + dim_len[0] % mpi_size; - /* !print *, mpi_rank, 'grid_xt', dim_len(3), grid_xt_start, grid_xt_loc_size */ + /* !print *, my_rank, 'grid_xt', dim_len(3), grid_xt_start, grid_xt_loc_size */ /* Size of local (i.e. for this pe) grid_yt data. */ grid_yt_loc_size = dim_len[1]/mpi_size; - grid_yt_start = mpi_rank * grid_yt_loc_size; - if (mpi_rank == mpi_size - 1) + grid_yt_start = my_rank * grid_yt_loc_size; + if (my_rank == mpi_size - 1) grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size; - /* !print *, mpi_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ + /* !print *, my_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ /* Size of local (i.e. for this pe) pfull data. */ pfull_loc_size = dim_len[2]/mpi_size; - pfull_start = mpi_rank * pfull_loc_size; - if (mpi_rank == mpi_size - 1) + pfull_start = my_rank * pfull_loc_size; + if (my_rank == mpi_size - 1) pfull_loc_size = pfull_loc_size + dim_len[2] % mpi_size; - /* !print *, mpi_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ + /* !print *, my_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ /* Size of local (i.e. for this pe) phalf data. */ phalf_loc_size = dim_len[3]/mpi_size; - phalf_start = mpi_rank * phalf_loc_size; - if (mpi_rank == mpi_size - 1) + phalf_start = my_rank * phalf_loc_size; + if (my_rank == mpi_size - 1) phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; - /* !print *, mpi_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ + /* !print *, my_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ /* specific to 4 pes. */ lon_xt_loc_size = 1536; lat_xt_loc_size = 1536; - if (mpi_rank == 0 || mpi_rank == 2) + if (my_rank == 0 || my_rank == 2) { lon_xt_start = 0; lat_xt_start = 0; @@ -120,7 +131,7 @@ main(int argc, char **argv) } lon_yt_loc_size = 768; lat_yt_loc_size = 768; - if (mpi_rank == 0 || mpi_rank == 1) + if (my_rank == 0 || my_rank == 1) { lon_yt_start = 0; lat_yt_start = 0; @@ -130,8 +141,8 @@ main(int argc, char **argv) lon_yt_start = 768; lat_yt_start = 768; } - /* ! print *, mpi_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ - /* ! print *, mpi_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ + /* ! print *, my_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ + /* ! print *, my_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ /* ! Allocate space on this pe to hold the data for this pe. */ if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; @@ -151,40 +162,41 @@ main(int argc, char **argv) /* Some fake data for this pe to write. */ for (i = 0; i < pfull_loc_size; i++) - value_pfull_loc[i] = mpi_rank * 100 + i; + value_pfull_loc[i] = my_rank * 100 + i; for (i = 0; i < phalf_loc_size; i++) - value_phalf_loc[i] = mpi_rank * 100 + i; + value_phalf_loc[i] = my_rank * 100 + i; for (i = 0; i < grid_xt_loc_size; i++) - value_grid_xt_loc[i] = mpi_rank * 100 + i; + value_grid_xt_loc[i] = my_rank * 100 + i; for (i = 0; i < grid_yt_loc_size; i++) - value_grid_yt_loc[i] = mpi_rank * 100 + i; + value_grid_yt_loc[i] = my_rank * 100 + i; for (j = 0; j < lon_yt_loc_size; j++) { for(i = 0; i < lon_xt_loc_size; i++) { - value_lon_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j; - value_lat_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j; + value_lon_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; + value_lat_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; for (k = 0; k < pfull_loc_size; k++) - value_clwmr_loc[j * lon_xt_loc_size + i] = mpi_rank * 100 + i + j + k; + value_clwmr_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j + k; } } - if (!mpi_rank) - printf("\n*** Testing parallel writes with compression filters.\n"); + /* if (!my_rank) */ + /* printf("\n*** Testing parallel writes with compression filters.\n"); */ { int s; for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) { - if (!mpi_rank) - { - printf("*** testing simple write with %s shuffle %d...", - (f ? "szip" : "zlib"), s); - } + /* if (!my_rank) */ + /* { */ + /* printf("*** testing simple write with %s shuffle %d...", */ + /* (f ? "szip" : "zlib"), s); */ + /* } */ /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ + ftime = MPI_Wtime(); if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; /* Turn off fill mode. */ @@ -241,7 +253,7 @@ main(int argc, char **argv) if (nc_enddef(ncid)) ERR; /* In NOAA code, do all processors write the single time value? */ - if (mpi_rank == 0) + if (my_rank == 0) if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; if (nc_redef(ncid)) ERR; @@ -273,7 +285,6 @@ main(int argc, char **argv) if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; if (nc_redef(ncid)) ERR; - /* Define variable clwmr and write data (?) */ dimid_data[0] = dimid[4]; dimid_data[1] = dimid[2]; @@ -312,6 +323,10 @@ main(int argc, char **argv) /* Close the file. */ if (nc_close(ncid)) ERR; + MPI_Barrier(MPI_COMM_WORLD); + write_us += (MPI_Wtime() - ftime) * MILLION; + if (my_rank == 0) + printf("%d: writing file took %d micro-seconds\n", my_rank, write_us); /* Check file. */ { @@ -360,7 +375,7 @@ main(int argc, char **argv) if (value_phalf_loc_in[i] != value_phalf_loc[i]) ERR; /* Check time. */ - if (mpi_rank == 0) + if (my_rank == 0) { if (nc_get_var_double(ncid, varid[6], &value_time_in)) ERR; if (value_time_in != value_time) ERR; @@ -395,14 +410,14 @@ main(int argc, char **argv) /* { */ /* if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; */ /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ - /* if (slab_data_in[i] != mpi_rank) ERR; */ + /* if (slab_data_in[i] != my_rank) ERR; */ /* } */ /* Close the netcdf file. */ if (nc_close(ncid)) ERR; } - if (!mpi_rank) + if (!my_rank) SUMMARIZE_ERR; } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ @@ -428,8 +443,8 @@ main(int argc, char **argv) /* Shut down MPI. */ MPI_Finalize(); - if (!mpi_rank) - FINAL_RESULTS; + /* if (!my_rank) */ + /* FINAL_RESULTS; */ return 0; } From 3508eba09ed2bdd230cf0bf23c30eee2109430f3 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 30 Jun 2020 13:36:57 -0600 Subject: [PATCH 22/50] better handling for multiple data vars --- nc_perf/bm_file.c | 2 +- nc_perf/tst_gfs_data_1.c | 88 +++++++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/nc_perf/bm_file.c b/nc_perf/bm_file.c index ee04cfdd7a..cfa2ab8db3 100644 --- a/nc_perf/bm_file.c +++ b/nc_perf/bm_file.c @@ -260,7 +260,7 @@ cmp_file(char *file1, char *file2, int *meta_read_us, size_t *data_read_us, #endif struct timeval start_time, end_time, diff_time; void *data = NULL, *data2 = NULL; - int a, v, d; + int a, v, d, dv; nc_type xtype, xtype2; int nvars, ndims, dimids[NC_MAX_VAR_DIMS], natts, real_ndims; int nvars2, ndims2, dimids2[NC_MAX_VAR_DIMS], natts2; diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 61cbd8165c..43d566e030 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -16,7 +16,7 @@ #include #define FILE_NAME "tst_gfs_data_1.nc" -#define NUM_VARS 8 +#define NUM_META_VARS 7 #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 @@ -29,6 +29,7 @@ #define NUM_COMPRESSION_FILTERS 1 #endif #define MILLION 1000000 +#define NUM_DATA_VARS 1 int main(int argc, char **argv) @@ -53,13 +54,13 @@ main(int argc, char **argv) int dimid_data[NDIM4]; /* Variables. */ - char var_name[NUM_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", - "lat", "pfull", "phalf", "time", - "clwmr"}; - int varid[NUM_VARS]; - int var_type[NUM_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, - NC_FLOAT, NC_FLOAT, NC_DOUBLE, NC_FLOAT}; - int var_ndims[NUM_VARS] = {1, 2, 1, 2, 1, 1, 1, 4}; + char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", + "lat", "pfull", "phalf", "time"}; + int varid[NUM_META_VARS]; + int data_varid[NUM_DATA_VARS]; + int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, + NC_FLOAT, NC_FLOAT, NC_DOUBLE}; + int var_ndims[NUM_META_VARS] = {1, 2, 1, 2, 1, 1, 1}; /* integer :: ideflate = 4 */ double value_time = 2.0; /* float value_time_in; */ @@ -79,7 +80,7 @@ main(int argc, char **argv) float *value_clwmr_loc, *value_clwmr_loc_in; int f; - int i, j, k; + int i, j, k, dv; int res; /* Initialize MPI. */ @@ -285,41 +286,54 @@ main(int argc, char **argv) if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; if (nc_redef(ncid)) ERR; - /* Define variable clwmr and write data (?) */ + /* Define dimensions for our data vars. */ dimid_data[0] = dimid[4]; dimid_data[1] = dimid[2]; dimid_data[2] = dimid[1]; dimid_data[3] = dimid[0]; - if (nc_def_var(ncid, var_name[7], var_type[7], NDIM4, dimid_data, &varid[7])) ERR; - /* Setting any filter only will work for HDF5-1.10.3 and later */ - /* versions. */ - if (!f) - res = nc_def_var_deflate(ncid, varid[7], s, 1, 4); - else - { - res = nc_def_var_deflate(ncid, varid[7], s, 0, 0); - if (!res) - res = nc_def_var_szip(ncid, varid[7], 32, 32); - } + /* Define data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + char data_var_name[NC_MAX_NAME + 1]; + + sprintf(data_var_name, "var_%d", dv); + if (nc_def_var(ncid, data_var_name, NC_DOUBLE, NDIM4, dimid_data, &data_varid[dv])) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); + else + { + res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); + } #ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; + if (res) ERR; #else - if (res != NC_EINVAL) ERR; + if (res != NC_EINVAL) ERR; #endif - if (nc_var_par_access(ncid, varid[7], NC_COLLECTIVE)) ERR; + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; if (nc_enddef(ncid)) ERR; - start[0] = 0; - start[1] = pfull_start; - start[2] = lat_yt_start; - start[3] = lat_xt_start; - count[0] = 1; - count[1] = pfull_loc_size; - count[2] = lat_yt_loc_size; - count[3] = lat_xt_loc_size; - if (nc_put_vara_float(ncid, varid[7], start, count, value_clwmr_loc)) ERR; - if (nc_redef(ncid)) ERR; + } + + /* Write one record each of the data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + start[0] = 0; + start[1] = pfull_start; + start[2] = lat_yt_start; + start[3] = lat_xt_start; + count[0] = 1; + count[1] = pfull_loc_size; + count[2] = lat_yt_loc_size; + count[3] = lat_xt_loc_size; + if (nc_put_vara_float(ncid, data_varid[dv], start, count, value_clwmr_loc)) ERR; + if (nc_redef(ncid)) ERR; + } /* Close the file. */ if (nc_close(ncid)) ERR; @@ -327,7 +341,7 @@ main(int argc, char **argv) write_us += (MPI_Wtime() - ftime) * MILLION; if (my_rank == 0) printf("%d: writing file took %d micro-seconds\n", my_rank, write_us); - + /* Check file. */ { int ndims, nvars, natts, unlimdimid; @@ -343,7 +357,7 @@ main(int argc, char **argv) /* Check file. */ if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; - if (ndims != NDIM5 || nvars != NUM_VARS || natts != 0 || unlimdimid != -1) ERR; + if (ndims != NDIM5 || nvars != NUM_META_VARS + NUM_DATA_VARS || natts != 0 || unlimdimid != -1) ERR; /* Check dims. */ for (d = 0; d < NDIM5; d++) @@ -355,7 +369,7 @@ main(int argc, char **argv) } /* Check vars. */ - for (v = 0; v < NUM_VARS; v++) + for (v = 0; v < NUM_META_VARS; v++) { int xtype_in, ndims_in, natts_in; int dimids_in[NDIM4]; From 9a75e1a4719e3457ff704944f3a5f464b8f084e0 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 30 Jun 2020 14:31:54 -0600 Subject: [PATCH 23/50] further test development --- nc_perf/tst_gfs_data_1.c | 158 +++++++-------------------------------- 1 file changed, 28 insertions(+), 130 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 43d566e030..bcc9980afd 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -28,8 +28,8 @@ #else #define NUM_COMPRESSION_FILTERS 1 #endif -#define MILLION 1000000 -#define NUM_DATA_VARS 1 +#define THOUSAND 1000 +#define NUM_DATA_VARS 10 int main(int argc, char **argv) @@ -40,8 +40,8 @@ main(int argc, char **argv) MPI_Info info = MPI_INFO_NULL; /* For timing. */ - double ftime; - int write_us; + double meta_start_time, meta_stop_time; + double data_start_time, data_stop_time; int ncid; size_t start[NDIM4], count[NDIM4]; @@ -60,24 +60,20 @@ main(int argc, char **argv) int data_varid[NUM_DATA_VARS]; int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE}; - int var_ndims[NUM_META_VARS] = {1, 2, 1, 2, 1, 1, 1}; - /* integer :: ideflate = 4 */ double value_time = 2.0; - /* float value_time_in; */ - /* real, allocatable :: value_clwmr(:,:,:,:) */ size_t pfull_loc_size, pfull_start; - float *value_pfull_loc, *value_pfull_loc_in; + float *value_pfull_loc; size_t phalf_loc_size, phalf_start; - float *value_phalf_loc, *value_phalf_loc_in; + float *value_phalf_loc; size_t grid_xt_loc_size, grid_xt_start; - double *value_grid_xt_loc, *value_grid_xt_loc_in; + double *value_grid_xt_loc; size_t grid_yt_loc_size, grid_yt_start; - double *value_grid_yt_loc, *value_grid_yt_loc_in; + double *value_grid_yt_loc; size_t lon_xt_loc_size, lon_xt_start, lon_yt_loc_size, lon_yt_start; - double *value_lon_loc, *value_lon_loc_in; + double *value_lon_loc; size_t lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start; - double *value_lat_loc, *value_lat_loc_in; - float *value_clwmr_loc, *value_clwmr_loc_in; + double *value_lat_loc; + float *value_clwmr_loc; int f; int i, j, k, dv; @@ -147,19 +143,12 @@ main(int argc, char **argv) /* ! Allocate space on this pe to hold the data for this pe. */ if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; - if (!(value_pfull_loc_in = malloc(pfull_loc_size * sizeof(float)))) ERR; if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; - if (!(value_phalf_loc_in = malloc(phalf_loc_size * sizeof(float)))) ERR; if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; - if (!(value_grid_xt_loc_in = malloc(grid_xt_loc_size * sizeof(double)))) ERR; if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; - if (!(value_grid_yt_loc_in = malloc(grid_yt_loc_size * sizeof(double)))) ERR; if (!(value_lon_loc = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lon_loc_in = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; if (!(value_lat_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lat_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; if (!(value_clwmr_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; - if (!(value_clwmr_loc_in = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; /* Some fake data for this pe to write. */ for (i = 0; i < pfull_loc_size; i++) @@ -197,7 +186,7 @@ main(int argc, char **argv) /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ - ftime = MPI_Wtime(); + meta_start_time = MPI_Wtime(); if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; /* Turn off fill mode. */ @@ -286,7 +275,7 @@ main(int argc, char **argv) if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; if (nc_redef(ncid)) ERR; - /* Define dimensions for our data vars. */ + /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; dimid_data[1] = dimid[2]; dimid_data[2] = dimid[1]; @@ -298,7 +287,7 @@ main(int argc, char **argv) char data_var_name[NC_MAX_NAME + 1]; sprintf(data_var_name, "var_%d", dv); - if (nc_def_var(ncid, data_var_name, NC_DOUBLE, NDIM4, dimid_data, &data_varid[dv])) ERR; + if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; /* Setting any filter only will work for HDF5-1.10.3 and later */ /* versions. */ @@ -315,11 +304,15 @@ main(int argc, char **argv) #else if (res != NC_EINVAL) ERR; #endif - - if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; - if (nc_enddef(ncid)) ERR; + + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; } + MPI_Barrier(MPI_COMM_WORLD); + meta_stop_time = MPI_Wtime(); + data_start_time = MPI_Wtime(); + /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { @@ -338,127 +331,32 @@ main(int argc, char **argv) /* Close the file. */ if (nc_close(ncid)) ERR; MPI_Barrier(MPI_COMM_WORLD); - write_us += (MPI_Wtime() - ftime) * MILLION; + data_stop_time = MPI_Wtime(); if (my_rank == 0) - printf("%d: writing file took %d micro-seconds\n", my_rank, write_us); + printf("meta %g data %g\n", meta_stop_time - meta_start_time, data_stop_time - data_start_time); - /* Check file. */ - { - int ndims, nvars, natts, unlimdimid; - char name_in[NC_MAX_NAME + 1]; - double value_time_in; - int d, v; - - /* int shuffle_in, deflate_in, deflate_level_in; */ - /* int options_mask_in, pixels_per_block_in; */ - - /* Reopen the file for parallel access. */ - if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; - - /* Check file. */ - if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; - if (ndims != NDIM5 || nvars != NUM_META_VARS + NUM_DATA_VARS || natts != 0 || unlimdimid != -1) ERR; - - /* Check dims. */ - for (d = 0; d < NDIM5; d++) - { - size_t len_in; - - if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR; - if (strcmp(name_in, dim_name[d]) && len_in != dim_len[d]) ERR; - } - - /* Check vars. */ - for (v = 0; v < NUM_META_VARS; v++) - { - int xtype_in, ndims_in, natts_in; - int dimids_in[NDIM4]; - - if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; - if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v] || ndims_in != var_ndims[v]) ERR; - } - - /* Check pfull data. */ - if (nc_get_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc_in)) ERR; - for (i = 0; i < pfull_loc_size; i++) - if (value_pfull_loc_in[i] != value_pfull_loc[i]) ERR; - - /* Check phalf data. */ - if (nc_get_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc_in)) ERR; - for (i = 0; i < phalf_loc_size; i++) - if (value_phalf_loc_in[i] != value_phalf_loc[i]) ERR; - - /* Check time. */ - if (my_rank == 0) - { - if (nc_get_var_double(ncid, varid[6], &value_time_in)) ERR; - if (value_time_in != value_time) ERR; - } - - /* Check grid_xt data. */ - if (nc_get_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc_in)) ERR; - for (i = 0; i < grid_xt_loc_size; i++) - if (value_grid_xt_loc_in[i] != value_grid_xt_loc[i]) ERR; - - /* Check grid_yt data. */ - if (nc_get_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc_in)) ERR; - for (i = 0; i < grid_yt_loc_size; i++) - if (value_grid_yt_loc_in[i] != value_grid_yt_loc[i]) ERR; - - /* /\* Check state of compression. *\/ */ - /* if (!f) */ - /* { */ - /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; */ - /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ - /* if (!deflate_in || deflate_level_in != 1) ERR; */ - /* } */ - /* else */ - /* { */ - /* if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; */ - /* if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; */ - /* if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; */ - /* } */ - - /* /\* Use parallel I/O to read the data. *\/ */ - /* for (start[2] = 0; start[2] < NUM_SLABS; start[2]++) */ - /* { */ - /* if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; */ - /* for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) */ - /* if (slab_data_in[i] != my_rank) ERR; */ - /* } */ - - /* Close the netcdf file. */ - if (nc_close(ncid)) ERR; - } - - if (!my_rank) - SUMMARIZE_ERR; } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ /* free(slab_data); */ /* Free resources. */ free(value_grid_xt_loc); - free(value_grid_xt_loc_in); free(value_grid_yt_loc); - free(value_grid_yt_loc_in); free(value_pfull_loc); - free(value_pfull_loc_in); free(value_phalf_loc); - free(value_phalf_loc_in); free(value_lon_loc); - free(value_lon_loc_in); free(value_lat_loc); - free(value_lat_loc_in); free(value_clwmr_loc); - free(value_clwmr_loc_in); } + if (!my_rank) + SUMMARIZE_ERR; + /* Shut down MPI. */ MPI_Finalize(); - /* if (!my_rank) */ - /* FINAL_RESULTS; */ + if (!my_rank) + FINAL_RESULTS; return 0; } From 1c8a361a62573a9b6bfde46d69ab013524010440 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 07:10:43 -0600 Subject: [PATCH 24/50] more benchmark development for tst_gfs_data_1.c --- nc_perf/tst_gfs_data_1.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index bcc9980afd..30ef7feb8d 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -20,11 +20,9 @@ #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 -#define NUM_SHUFFLE_SETTINGS 1 -/* #define NUM_SHUFFLE_SETTINGS 2 */ +#define NUM_SHUFFLE_SETTINGS 2 #ifdef HAVE_H5Z_SZIP -#define NUM_COMPRESSION_FILTERS 1 -/* #define NUM_COMPRESSION_FILTERS 2 */ +#define NUM_COMPRESSION_FILTERS 2 #else #define NUM_COMPRESSION_FILTERS 1 #endif @@ -170,20 +168,17 @@ main(int argc, char **argv) } } - /* if (!my_rank) */ - /* printf("\n*** Testing parallel writes with compression filters.\n"); */ + if (my_rank == 0) + { + printf("Benchmarking creation of UFS file.\n"); + printf("comp, shuffle, meta, data\n"); + } { int s; for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) { - /* if (!my_rank) */ - /* { */ - /* printf("*** testing simple write with %s shuffle %d...", */ - /* (f ? "szip" : "zlib"), s); */ - /* } */ - /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ meta_start_time = MPI_Wtime(); @@ -333,7 +328,7 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); data_stop_time = MPI_Wtime(); if (my_rank == 0) - printf("meta %g data %g\n", meta_stop_time - meta_start_time, data_stop_time - data_start_time); + printf("%s, %d, %g, %g\n", (f ? "szip" : "zlib"), s, meta_stop_time - meta_start_time, data_stop_time - data_start_time); } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ From 010c1b2766399dcd8af4a9285c73ebe6de6cb02e Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 08:28:58 -0600 Subject: [PATCH 25/50] fixed Makefile to refrain from building nc_perf/tst_gfs_data_1 for non-parallel builds --- nc_perf/Makefile.am | 4 +- nc_perf/tst_gfs_data_1.c | 207 +++++++++++++++++++++------------------ nc_perf/tst_mem.c | 4 +- 3 files changed, 113 insertions(+), 102 deletions(-) diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am index 898cca8b02..944179788b 100644 --- a/nc_perf/Makefile.am +++ b/nc_perf/Makefile.am @@ -20,8 +20,7 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la check_PROGRAMS = tst_create_files bm_file tst_chunks3 tst_ar4 \ tst_ar4_3d tst_ar4_4d bm_many_objs tst_h_many_atts bm_many_atts \ tst_files2 tst_files3 tst_mem tst_mem1 tst_knmi bm_netcdf4_recs \ -tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando \ -tst_gfs_data_1 +tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando bm_file_SOURCES = bm_file.c tst_utils.c bm_netcdf4_recs_SOURCES = bm_netcdf4_recs.c tst_utils.c @@ -51,6 +50,7 @@ run_bm_test2.log: tst_create_files.log # This will run parallel I/O benchmarks for parallel builds. if TEST_PARALLEL4 +check_PROGRAMS += tst_gfs_data_1 TESTS += run_par_bm_test.sh run_gfs_test.sh run_par_bm_test.log: tst_create_files.log run_bm_test1.log endif # TEST_PARALLEL4 diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 30ef7feb8d..697b377265 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -29,6 +29,12 @@ #define THOUSAND 1000 #define NUM_DATA_VARS 10 +int +write_metadata(int ncid) +{ + return 0; +} + int main(int argc, char **argv) { @@ -184,124 +190,129 @@ main(int argc, char **argv) meta_start_time = MPI_Wtime(); if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - /* Turn off fill mode. */ - if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + if (write_metadata(ncid)) ERR; + + { + + /* Turn off fill mode. */ + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; - /* Define dimension grid_xt. */ - if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; + /* Define dimension grid_xt. */ + if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; - /* Define dimension grid_yt. */ - if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; + /* Define dimension grid_yt. */ + if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; - /* Define variable grid_xt. */ - if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; - if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; + /* Define variable grid_xt. */ + if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; + if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; - /* Define variable lon. */ - if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; - if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); + /* Define variable lon. */ + if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; + if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); - /* Define variable grid_yt. */ - if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; - if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; + /* Define variable grid_yt. */ + if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; + if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; - /* Define variable lat. */ - if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; - if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; + /* Define variable lat. */ + if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; + if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; - /* Define dimension pfull. */ - if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; + /* Define dimension pfull. */ + if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; - /* Define variable pfull and write data. */ - if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; - if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc)) ERR; - if (nc_redef(ncid)) ERR; + /* Define variable pfull and write data. */ + if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; + if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc)) ERR; + if (nc_redef(ncid)) ERR; - /* Define dimension phalf. */ - if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; + /* Define dimension phalf. */ + if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; - /* Define variable phalf and write data. */ - if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; - if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; - if (nc_redef(ncid)) ERR; + /* Define variable phalf and write data. */ + if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; + if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_redef(ncid)) ERR; - /* Define dimension time. */ - if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; + /* Define dimension time. */ + if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; - /* Define variable time and write data. */ - if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; - if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; + /* Define variable time and write data. */ + if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; + if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; - /* In NOAA code, do all processors write the single time value? */ - if (my_rank == 0) - if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; - if (nc_redef(ncid)) ERR; - - /* Write variable grid_xt data. */ - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write lon data. */ - if (nc_enddef(ncid)) ERR; - start[0] = lon_xt_start; - start[1] = lon_yt_start; - count[0] = lon_xt_loc_size; - count[1] = lon_yt_loc_size; - if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write grid_yt data. */ - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write lat data. */ - if (nc_enddef(ncid)) ERR; - start[0] = lat_xt_start; - start[1] = lat_yt_start; - count[0] = lat_xt_loc_size; - count[1] = lat_yt_loc_size; - if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Specify dimensions for our data vars. */ - dimid_data[0] = dimid[4]; - dimid_data[1] = dimid[2]; - dimid_data[2] = dimid[1]; - dimid_data[3] = dimid[0]; - - /* Define data variables. */ - for (dv = 0; dv < NUM_DATA_VARS; dv++) - { - char data_var_name[NC_MAX_NAME + 1]; + /* In NOAA code, do all processors write the single time value? */ + if (my_rank == 0) + if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; + if (nc_redef(ncid)) ERR; + + /* Write variable grid_xt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lon data. */ + if (nc_enddef(ncid)) ERR; + start[0] = lon_xt_start; + start[1] = lon_yt_start; + count[0] = lon_xt_loc_size; + count[1] = lon_yt_loc_size; + if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write grid_yt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lat data. */ + if (nc_enddef(ncid)) ERR; + start[0] = lat_xt_start; + start[1] = lat_yt_start; + count[0] = lat_xt_loc_size; + count[1] = lat_yt_loc_size; + if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; + if (nc_redef(ncid)) ERR; - sprintf(data_var_name, "var_%d", dv); - if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; + /* Specify dimensions for our data vars. */ + dimid_data[0] = dimid[4]; + dimid_data[1] = dimid[2]; + dimid_data[2] = dimid[1]; + dimid_data[3] = dimid[0]; - /* Setting any filter only will work for HDF5-1.10.3 and later */ - /* versions. */ - if (!f) - res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); - else + /* Define data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) { - res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); - if (!res) - res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); - } + char data_var_name[NC_MAX_NAME + 1]; + + sprintf(data_var_name, "var_%d", dv); + if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); + else + { + res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); + } #ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; + if (res) ERR; #else - if (res != NC_EINVAL) ERR; + if (res != NC_EINVAL) ERR; #endif - if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; - if (nc_enddef(ncid)) ERR; + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; + } } MPI_Barrier(MPI_COMM_WORLD); diff --git a/nc_perf/tst_mem.c b/nc_perf/tst_mem.c index 6e03bc12e2..a22a2f0960 100644 --- a/nc_perf/tst_mem.c +++ b/nc_perf/tst_mem.c @@ -2,8 +2,8 @@ Copyright 2018 University Corporation for Atmospheric Research/Unidata See COPYRIGHT file for conditions of use. - Test internal netcdf-4 file code. - $Id$ + Test memory use of a netCDF-4 file with unlimited dimensions. + Ed Hartnett */ #include From 0568ca5e67dcbb3975126f0c47aacf4e3727e1e5 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 09:46:47 -0600 Subject: [PATCH 26/50] move data decomposition calculations to decomp_4D() function --- nc_perf/tst_gfs_data_1.c | 51 ++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 697b377265..4ffdde0f03 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -20,14 +20,16 @@ #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 -#define NUM_SHUFFLE_SETTINGS 2 +#define NUM_SHUFFLE_SETTINGS 1 +/* #define NUM_SHUFFLE_SETTINGS 2 */ #ifdef HAVE_H5Z_SZIP -#define NUM_COMPRESSION_FILTERS 2 +/* #define NUM_COMPRESSION_FILTERS 2 */ +#define NUM_COMPRESSION_FILTERS 1 #else #define NUM_COMPRESSION_FILTERS 1 #endif #define THOUSAND 1000 -#define NUM_DATA_VARS 10 +#define NUM_DATA_VARS 1 int write_metadata(int ncid) @@ -35,6 +37,33 @@ write_metadata(int ncid) return 0; } +int +decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) +{ + start[0] = 0; + count[0] = 1; + count[1] = dim_len[2]/mpi_size; + start[1] = my_rank * count[1]; + + if (my_rank == 0 || my_rank == 1) + { + start[2] = 0; + start[3] = 0; + } + else + { + start[2] = 768; + start[3] = 768; + } + count[2] = 768; + count[3] = 1536; + + printf("%d: start %ld %ld %ld %ld count %ld %ld %ld %ld\n", my_rank, start[0], + start[1], start[2], start[3], count[0], count[1], count[2], count[3]); + + return 0; +} + int main(int argc, char **argv) { @@ -49,6 +78,7 @@ main(int argc, char **argv) int ncid; size_t start[NDIM4], count[NDIM4]; + size_t data_start[NDIM4], data_count[NDIM4]; /* Dimensions. */ char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", @@ -88,6 +118,9 @@ main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + /* Determine data decomposition. */ + if (decomp_4D(my_rank, mpi_size, dim_len, data_start, data_count)) ERR; + /* Size of local (i.e. for this pe) grid_xt data. */ grid_xt_loc_size = dim_len[0]/mpi_size; grid_xt_start = my_rank * grid_xt_loc_size; @@ -318,19 +351,11 @@ main(int argc, char **argv) MPI_Barrier(MPI_COMM_WORLD); meta_stop_time = MPI_Wtime(); data_start_time = MPI_Wtime(); - + /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { - start[0] = 0; - start[1] = pfull_start; - start[2] = lat_yt_start; - start[3] = lat_xt_start; - count[0] = 1; - count[1] = pfull_loc_size; - count[2] = lat_yt_loc_size; - count[3] = lat_xt_loc_size; - if (nc_put_vara_float(ncid, data_varid[dv], start, count, value_clwmr_loc)) ERR; + if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_clwmr_loc)) ERR; if (nc_redef(ncid)) ERR; } From 3f6a367381780ca796fda2bca33e4fe8053c058f Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 12:26:12 -0600 Subject: [PATCH 27/50] further development --- nc_perf/Makefile.am | 2 +- nc_perf/gfs_sample.cdl | 274 +++++++++++++++++++++++++++++++++++++++ nc_perf/tst_gfs_data_1.c | 54 ++++---- 3 files changed, 299 insertions(+), 31 deletions(-) create mode 100644 nc_perf/gfs_sample.cdl diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am index 944179788b..6516d7b645 100644 --- a/nc_perf/Makefile.am +++ b/nc_perf/Makefile.am @@ -60,7 +60,7 @@ endif # BUILD_UTILITIES # because configure substitute in the launcher (usually mpiexec). EXTRA_DIST = run_knmi_bm.sh perftest.sh run_bm_test1.sh \ run_bm_test2.sh run_tst_chunks.sh run_bm_elena.sh CMakeLists.txt \ -run_gfs_test.sh.in run_par_bm_test.sh.in +run_gfs_test.sh.in run_par_bm_test.sh.in gfs_sample.cdl CLEANFILES = tst_*.nc bigmeta.nc bigvars.nc floats*.nc floats*.cdl \ shorts*.nc shorts*.cdl ints*.nc ints*.cdl tst_*.cdl diff --git a/nc_perf/gfs_sample.cdl b/nc_perf/gfs_sample.cdl new file mode 100644 index 0000000000..f44a71b82e --- /dev/null +++ b/nc_perf/gfs_sample.cdl @@ -0,0 +1,274 @@ +netcdf gfs.t00z.atmf024 { +dimensions: + grid_xt = 3072 ; + grid_yt = 1536 ; + pfull = 127 ; + phalf = 128 ; + time = UNLIMITED ; // (1 currently) +variables: + double grid_xt(grid_xt) ; + grid_xt:cartesian_axis = "X" ; + grid_xt:long_name = "T-cell longitude" ; + grid_xt:units = "degrees_E" ; + grid_xt:_Storage = "contiguous" ; + grid_xt:_Endianness = "little" ; + double lon(grid_yt, grid_xt) ; + lon:long_name = "T-cell longitude" ; + lon:units = "degrees_E" ; + lon:_Storage = "contiguous" ; + lon:_Endianness = "little" ; + double grid_yt(grid_yt) ; + grid_yt:cartesian_axis = "Y" ; + grid_yt:long_name = "T-cell latiitude" ; + grid_yt:units = "degrees_N" ; + grid_yt:_Storage = "contiguous" ; + grid_yt:_Endianness = "little" ; + double lat(grid_yt, grid_xt) ; + lat:long_name = "T-cell latitude" ; + lat:units = "degrees_N" ; + lat:_Storage = "contiguous" ; + lat:_Endianness = "little" ; + float pfull(pfull) ; + pfull:long_name = "ref full pressure level" ; + pfull:units = "mb" ; + pfull:cartesian_axis = "Z" ; + pfull:positive = "down" ; + pfull:edges = "phalf" ; + pfull:_Storage = "contiguous" ; + pfull:_Endianness = "little" ; + float phalf(phalf) ; + phalf:long_name = "ref half pressure level" ; + phalf:units = "mb" ; + phalf:cartesian_axis = "Z" ; + phalf:positive = "down" ; + phalf:_Storage = "contiguous" ; + phalf:_Endianness = "little" ; + double time(time) ; + time:long_name = "time" ; + time:units = "hours since 2018-01-10 00:00:00" ; + time:cartesian_axis = "T" ; + time:calendar_type = "JULIAN" ; + time:calendar = "JULIAN" ; + time:_Storage = "chunked" ; + time:_ChunkSizes = 1 ; + time:_Endianness = "little" ; + float cld_amt(time, pfull, grid_yt, grid_xt) ; + cld_amt:long_name = "cloud amount" ; + cld_amt:units = "1" ; + cld_amt:missing_value = -1.e+10f ; + cld_amt:_FillValue = -1.e+10f ; + cld_amt:cell_methods = "time: point" ; + cld_amt:output_file = "atm" ; + cld_amt:max_abs_compression_error = 3.057718e-05f ; + cld_amt:nbits = 14 ; + cld_amt:_Storage = "chunked" ; + cld_amt:_ChunkSizes = 1, 22, 308, 615 ; + cld_amt:_DeflateLevel = 1 ; + cld_amt:_Endianness = "little" ; + float clwmr(time, pfull, grid_yt, grid_xt) ; + clwmr:long_name = "cloud water mixing ratio" ; + clwmr:units = "kg/kg" ; + clwmr:missing_value = -1.e+10f ; + clwmr:_FillValue = -1.e+10f ; + clwmr:cell_methods = "time: point" ; + clwmr:output_file = "atm" ; + clwmr:max_abs_compression_error = 4.976755e-08f ; + clwmr:nbits = 14 ; + clwmr:_Storage = "chunked" ; + clwmr:_ChunkSizes = 1, 22, 308, 615 ; + clwmr:_DeflateLevel = 1 ; + clwmr:_Endianness = "little" ; + float delz(time, pfull, grid_yt, grid_xt) ; + delz:long_name = "height thickness" ; + delz:units = "m" ; + delz:missing_value = -1.e+10f ; + delz:_FillValue = -1.e+10f ; + delz:cell_methods = "time: point" ; + delz:output_file = "atm" ; + delz:max_abs_compression_error = 0.1002197f ; + delz:nbits = 14 ; + delz:_Storage = "chunked" ; + delz:_ChunkSizes = 1, 22, 308, 615 ; + delz:_DeflateLevel = 1 ; + delz:_Endianness = "little" ; + float dpres(time, pfull, grid_yt, grid_xt) ; + dpres:long_name = "pressure thickness" ; + dpres:units = "pa" ; + dpres:missing_value = -1.e+10f ; + dpres:_FillValue = -1.e+10f ; + dpres:cell_methods = "time: point" ; + dpres:output_file = "atm" ; + dpres:max_abs_compression_error = 0.05603027f ; + dpres:nbits = 14 ; + dpres:_Storage = "chunked" ; + dpres:_ChunkSizes = 1, 22, 308, 615 ; + dpres:_DeflateLevel = 1 ; + dpres:_Endianness = "little" ; + float dzdt(time, pfull, grid_yt, grid_xt) ; + dzdt:long_name = "vertical wind" ; + dzdt:units = "m/sec" ; + dzdt:missing_value = -1.e+10f ; + dzdt:_FillValue = -1.e+10f ; + dzdt:cell_methods = "time: point" ; + dzdt:output_file = "atm" ; + dzdt:max_abs_compression_error = 0.0003833771f ; + dzdt:nbits = 14 ; + dzdt:_Storage = "chunked" ; + dzdt:_ChunkSizes = 1, 22, 308, 615 ; + dzdt:_DeflateLevel = 1 ; + dzdt:_Endianness = "little" ; + float grle(time, pfull, grid_yt, grid_xt) ; + grle:long_name = "graupel mixing ratio" ; + grle:units = "kg/kg" ; + grle:missing_value = -1.e+10f ; + grle:_FillValue = -1.e+10f ; + grle:cell_methods = "time: point" ; + grle:output_file = "atm" ; + grle:max_abs_compression_error = 3.105961e-07f ; + grle:nbits = 14 ; + grle:_Storage = "chunked" ; + grle:_ChunkSizes = 1, 22, 308, 615 ; + grle:_DeflateLevel = 1 ; + grle:_Endianness = "little" ; + float hgtsfc(time, grid_yt, grid_xt) ; + hgtsfc:long_name = "surface geopotential height" ; + hgtsfc:units = "gpm" ; + hgtsfc:missing_value = -1.e+10f ; + hgtsfc:_FillValue = -1.e+10f ; + hgtsfc:cell_methods = "time: point" ; + hgtsfc:output_file = "atm" ; + hgtsfc:_Storage = "chunked" ; + hgtsfc:_ChunkSizes = 1, 768, 1536 ; + hgtsfc:_DeflateLevel = 1 ; + hgtsfc:_Shuffle = "true" ; + hgtsfc:_Endianness = "little" ; + float icmr(time, pfull, grid_yt, grid_xt) ; + icmr:long_name = "cloud ice mixing ratio" ; + icmr:units = "kg/kg" ; + icmr:missing_value = -1.e+10f ; + icmr:_FillValue = -1.e+10f ; + icmr:cell_methods = "time: point" ; + icmr:output_file = "atm" ; + icmr:max_abs_compression_error = 4.316098e-08f ; + icmr:nbits = 14 ; + icmr:_Storage = "chunked" ; + icmr:_ChunkSizes = 1, 22, 308, 615 ; + icmr:_DeflateLevel = 1 ; + icmr:_Endianness = "little" ; + float o3mr(time, pfull, grid_yt, grid_xt) ; + o3mr:long_name = "ozone mixing ratio" ; + o3mr:units = "kg/kg" ; + o3mr:missing_value = -1.e+10f ; + o3mr:_FillValue = -1.e+10f ; + o3mr:cell_methods = "time: point" ; + o3mr:output_file = "atm" ; + o3mr:max_abs_compression_error = 5.438778e-10f ; + o3mr:nbits = 14 ; + o3mr:_Storage = "chunked" ; + o3mr:_ChunkSizes = 1, 22, 308, 615 ; + o3mr:_DeflateLevel = 1 ; + o3mr:_Endianness = "little" ; + float pressfc(time, grid_yt, grid_xt) ; + pressfc:long_name = "surface pressure" ; + pressfc:units = "pa" ; + pressfc:missing_value = -1.e+10f ; + pressfc:_FillValue = -1.e+10f ; + pressfc:cell_methods = "time: point" ; + pressfc:output_file = "atm" ; + pressfc:_Storage = "chunked" ; + pressfc:_ChunkSizes = 1, 768, 1536 ; + pressfc:_DeflateLevel = 1 ; + pressfc:_Shuffle = "true" ; + pressfc:_Endianness = "little" ; + float rwmr(time, pfull, grid_yt, grid_xt) ; + rwmr:long_name = "rain mixing ratio" ; + rwmr:units = "kg/kg" ; + rwmr:missing_value = -1.e+10f ; + rwmr:_FillValue = -1.e+10f ; + rwmr:cell_methods = "time: point" ; + rwmr:output_file = "atm" ; + rwmr:max_abs_compression_error = 1.406297e-07f ; + rwmr:nbits = 14 ; + rwmr:_Storage = "chunked" ; + rwmr:_ChunkSizes = 1, 22, 308, 615 ; + rwmr:_DeflateLevel = 1 ; + rwmr:_Endianness = "little" ; + float snmr(time, pfull, grid_yt, grid_xt) ; + snmr:long_name = "snow mixing ratio" ; + snmr:units = "kg/kg" ; + snmr:missing_value = -1.e+10f ; + snmr:_FillValue = -1.e+10f ; + snmr:cell_methods = "time: point" ; + snmr:output_file = "atm" ; + snmr:max_abs_compression_error = 6.280607e-08f ; + snmr:nbits = 14 ; + snmr:_Storage = "chunked" ; + snmr:_ChunkSizes = 1, 22, 308, 615 ; + snmr:_DeflateLevel = 1 ; + snmr:_Endianness = "little" ; + float spfh(time, pfull, grid_yt, grid_xt) ; + spfh:long_name = "specific humidity" ; + spfh:units = "kg/kg" ; + spfh:missing_value = -1.e+10f ; + spfh:_FillValue = -1.e+10f ; + spfh:cell_methods = "time: point" ; + spfh:output_file = "atm" ; + spfh:max_abs_compression_error = 7.404014e-07f ; + spfh:nbits = 14 ; + spfh:_Storage = "chunked" ; + spfh:_ChunkSizes = 1, 22, 308, 615 ; + spfh:_DeflateLevel = 1 ; + spfh:_Endianness = "little" ; + float tmp(time, pfull, grid_yt, grid_xt) ; + tmp:long_name = "temperature" ; + tmp:units = "K" ; + tmp:missing_value = -1.e+10f ; + tmp:_FillValue = -1.e+10f ; + tmp:cell_methods = "time: point" ; + tmp:output_file = "atm" ; + tmp:max_abs_compression_error = 0.004516602f ; + tmp:nbits = 14 ; + tmp:_Storage = "chunked" ; + tmp:_ChunkSizes = 1, 22, 308, 615 ; + tmp:_DeflateLevel = 1 ; + tmp:_Endianness = "little" ; + float ugrd(time, pfull, grid_yt, grid_xt) ; + ugrd:long_name = "zonal wind" ; + ugrd:units = "m/sec" ; + ugrd:missing_value = -1.e+10f ; + ugrd:_FillValue = -1.e+10f ; + ugrd:cell_methods = "time: point" ; + ugrd:output_file = "atm" ; + ugrd:max_abs_compression_error = 0.008621216f ; + ugrd:nbits = 14 ; + ugrd:_Storage = "chunked" ; + ugrd:_ChunkSizes = 1, 22, 308, 615 ; + ugrd:_DeflateLevel = 1 ; + ugrd:_Endianness = "little" ; + float vgrd(time, pfull, grid_yt, grid_xt) ; + vgrd:long_name = "meridional wind" ; + vgrd:units = "m/sec" ; + vgrd:missing_value = -1.e+10f ; + vgrd:_FillValue = -1.e+10f ; + vgrd:cell_methods = "time: point" ; + vgrd:output_file = "atm" ; + vgrd:max_abs_compression_error = 0.00667572f ; + vgrd:nbits = 14 ; + vgrd:_Storage = "chunked" ; + vgrd:_ChunkSizes = 1, 22, 308, 615 ; + vgrd:_DeflateLevel = 1 ; + vgrd:_Endianness = "little" ; + +// global attributes: + :hydrostatic = "non-hydrostatic" ; + :ncnsto = 9 ; + :ak = 0.999f, 1.605f, 2.532f, 3.924f, 5.976f, 8.947f, 13.177f, 19.096f, 27.243f, 38.276f, 52.984f, 72.293f, 97.269f, 129.11f, 169.135f, 218.767f, 279.506f, 352.894f, 440.481f, 543.782f, 664.236f, 803.164f, 961.734f, 1140.931f, 1341.538f, 1564.119f, 1809.028f, 2076.415f, 2366.252f, 2678.372f, 3012.51f, 3368.363f, 3745.646f, 4144.164f, 4563.881f, 5004.995f, 5468.017f, 5953.848f, 6463.864f, 7000.f, 7563.494f, 8150.661f, 8756.529f, 9376.141f, 10004.55f, 10636.85f, 11268.16f, 11893.64f, 12508.52f, 13108.09f, 13687.73f, 14242.89f, 14769.15f, 15262.2f, 15717.86f, 16132.09f, 16501.02f, 16820.94f, 17088.32f, 17299.85f, 17453.08f, 17548.35f, 17586.77f, 17569.7f, 17498.7f, 17375.56f, 17202.3f, 16981.14f, 16714.5f, 16405.02f, 16055.49f, 15668.86f, 15248.25f, 14796.87f, 14318.04f, 13815.15f, 13291.63f, 12750.92f, 12196.47f, 11631.66f, 11059.83f, 10484.21f, 9907.927f, 9333.967f, 8765.155f, 8204.142f, 7653.387f, 7115.147f, 6591.468f, 6084.176f, 5594.876f, 5124.949f, 4675.554f, 4247.633f, 3841.918f, 3458.933f, 3099.01f, 2762.297f, 2448.768f, 2158.238f, 1890.375f, 1644.712f, 1420.661f, 1217.528f, 1034.524f, 870.778f, 725.348f, 597.235f, 485.392f, 388.734f, 306.149f, 236.502f, 178.651f, 131.447f, 93.74f, 64.392f, 42.274f, 26.274f, 15.302f, 8.287f, 4.19f, 1.994f, 0.81f, 0.232f, 0.029f, 0.f, 0.f, 0.f ; + :bk = 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 1.018e-05f, 8.141e-05f, 0.00027469f, 0.00065078f, 0.00127009f, 0.00219248f, 0.00347713f, 0.00518228f, 0.00736504f, 0.0100812f, 0.01338492f, 0.01732857f, 0.02196239f, 0.02733428f, 0.03348954f, 0.04047056f, 0.04831661f, 0.05706358f, 0.06674372f, 0.07738548f, 0.08900629f, 0.101594f, 0.1151262f, 0.1295762f, 0.1449129f, 0.1611008f, 0.1780999f, 0.195866f, 0.2143511f, 0.2335031f, 0.2532663f, 0.2735822f, 0.294389f, 0.3156229f, 0.337218f, 0.3591072f, 0.3812224f, 0.4034951f, 0.4258572f, 0.4482413f, 0.4705813f, 0.492813f, 0.5148743f, 0.5367062f, 0.5582525f, 0.5794605f, 0.6002815f, 0.6206707f, 0.6405875f, 0.6599957f, 0.6788633f, 0.6971631f, 0.714872f, 0.7319713f, 0.7484465f, 0.7642871f, 0.7794867f, 0.7940422f, 0.8079541f, 0.8212263f, 0.8338652f, 0.8458801f, 0.8572826f, 0.8680866f, 0.8783077f, 0.8879632f, 0.8970718f, 0.9056532f, 0.9137284f, 0.9213187f, 0.9284464f, 0.9351338f, 0.9414037f, 0.9472789f, 0.9527821f, 0.957936f, 0.962763f, 0.9672851f, 0.971524f, 0.9755009f, 0.9792364f, 0.9827508f, 0.9860625f, 0.9891851f, 0.9921299f, 0.9949077f, 0.9975282f, 1.f ; + :source = "FV3GFS" ; + :grid = "gaussian" ; + :im = 3072 ; + :jm = 1536 ; + :_SuperblockVersion = 2 ; + :_IsNetcdf4 = 0 ; + :_Format = "netCDF-4 classic model" ; +} diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 4ffdde0f03..3044fe1237 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -32,7 +32,7 @@ #define NUM_DATA_VARS 1 int -write_metadata(int ncid) +write_metadata(int ncid, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc) { return 0; } @@ -42,8 +42,12 @@ decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) { start[0] = 0; count[0] = 1; + count[1] = dim_len[2]/mpi_size; start[1] = my_rank * count[1]; + /* Add any extra to the end. */ + if (my_rank == mpi_size - 1) + count[1] = count[1] + dim_len[2] % mpi_size; if (my_rank == 0 || my_rank == 1) { @@ -80,9 +84,6 @@ main(int argc, char **argv) size_t start[NDIM4], count[NDIM4]; size_t data_start[NDIM4], data_count[NDIM4]; - /* Dimensions. */ - char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", - "phalf", "time"}; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; int dimid[NDIM5]; int dimid_data[NDIM4]; @@ -95,7 +96,6 @@ main(int argc, char **argv) int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE}; double value_time = 2.0; - size_t pfull_loc_size, pfull_start; float *value_pfull_loc; size_t phalf_loc_size, phalf_start; float *value_phalf_loc; @@ -107,7 +107,7 @@ main(int argc, char **argv) double *value_lon_loc; size_t lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start; double *value_lat_loc; - float *value_clwmr_loc; + float *value_data; int f; int i, j, k, dv; @@ -126,28 +126,18 @@ main(int argc, char **argv) grid_xt_start = my_rank * grid_xt_loc_size; if (my_rank == mpi_size - 1) grid_xt_loc_size = grid_xt_loc_size + dim_len[0] % mpi_size; - /* !print *, my_rank, 'grid_xt', dim_len(3), grid_xt_start, grid_xt_loc_size */ /* Size of local (i.e. for this pe) grid_yt data. */ grid_yt_loc_size = dim_len[1]/mpi_size; grid_yt_start = my_rank * grid_yt_loc_size; if (my_rank == mpi_size - 1) grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size; - /* !print *, my_rank, 'grid_yt', dim_len(3), grid_yt_start, grid_yt_loc_size */ - - /* Size of local (i.e. for this pe) pfull data. */ - pfull_loc_size = dim_len[2]/mpi_size; - pfull_start = my_rank * pfull_loc_size; - if (my_rank == mpi_size - 1) - pfull_loc_size = pfull_loc_size + dim_len[2] % mpi_size; - /* !print *, my_rank, 'pfull', dim_len(3), pfull_start, pfull_loc_size */ /* Size of local (i.e. for this pe) phalf data. */ phalf_loc_size = dim_len[3]/mpi_size; phalf_start = my_rank * phalf_loc_size; if (my_rank == mpi_size - 1) phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; - /* !print *, my_rank, 'phalf', dim_len(4), phalf_start, phalf_loc_size */ /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ /* specific to 4 pes. */ @@ -175,20 +165,20 @@ main(int argc, char **argv) lon_yt_start = 768; lat_yt_start = 768; } - /* ! print *, my_rank, 'lon_xt_start', lon_xt_start, 'lon_yt_start', lon_yt_start */ - /* ! print *, my_rank, 'lon_xt_loc_size', lon_xt_loc_size, 'lon_yt_loc_size', lon_yt_loc_size */ - /* ! Allocate space on this pe to hold the data for this pe. */ - if (!(value_pfull_loc = malloc(pfull_loc_size * sizeof(float)))) ERR; + /* Allocate space on this pe to hold the coordinate var data for this pe. */ + if (!(value_pfull_loc = malloc(data_count[1] * sizeof(float)))) ERR; if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; if (!(value_lon_loc = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; if (!(value_lat_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; - if (!(value_clwmr_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * pfull_loc_size * sizeof(float)))) ERR; + + /* Allocate space to hold the data. */ + if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; /* Some fake data for this pe to write. */ - for (i = 0; i < pfull_loc_size; i++) + for (i = 0; i < data_count[1]; i++) value_pfull_loc[i] = my_rank * 100 + i; for (i = 0; i < phalf_loc_size; i++) value_phalf_loc[i] = my_rank * 100 + i; @@ -202,8 +192,8 @@ main(int argc, char **argv) { value_lon_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; value_lat_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; - for (k = 0; k < pfull_loc_size; k++) - value_clwmr_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j + k; + for (k = 0; k < data_count[1]; k++) + value_data[j * lon_xt_loc_size + i] = my_rank * 100 + i + j + k; } } @@ -223,10 +213,14 @@ main(int argc, char **argv) meta_start_time = MPI_Wtime(); if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - if (write_metadata(ncid)) ERR; + if (write_metadata(ncid, dim_len, phalf_loc_size, phalf_start, value_phalf_loc)) ERR; { + /* Dimensions. */ + char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", + "phalf", "time"}; + /* Turn off fill mode. */ if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; @@ -259,13 +253,13 @@ main(int argc, char **argv) if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_loc_size, value_pfull_loc)) ERR; + if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; if (nc_redef(ncid)) ERR; - /* Define dimension phalf. */ + /* Define dimension phalf. This dim is only used by the phalf coord var. */ if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; - /* Define variable phalf and write data. */ + /* Define coord variable phalf and write data. */ if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; @@ -355,7 +349,7 @@ main(int argc, char **argv) /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { - if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_clwmr_loc)) ERR; + if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)) ERR; if (nc_redef(ncid)) ERR; } @@ -377,7 +371,7 @@ main(int argc, char **argv) free(value_phalf_loc); free(value_lon_loc); free(value_lat_loc); - free(value_clwmr_loc); + free(value_data); } if (!my_rank) From 527589a0d7b611ad08268a0368dc4ea9db5ce4f9 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 13:30:24 -0600 Subject: [PATCH 28/50] now create metadata in a function with many parameters --- nc_perf/tst_gfs_data_1.c | 320 ++++++++++++++++++++------------------- 1 file changed, 161 insertions(+), 159 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 3044fe1237..1c76a348bf 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -17,6 +17,7 @@ #define FILE_NAME "tst_gfs_data_1.nc" #define NUM_META_VARS 7 +#define NDIM2 2 #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 @@ -32,8 +33,143 @@ #define NUM_DATA_VARS 1 int -write_metadata(int ncid, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc) +write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, + size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, size_t grid_yt_loc_size, double *value_grid_yt_loc, + size_t *lat_start, size_t *lat_count, double *value_lat_loc, size_t *lon_start, size_t *lon_count, double *value_lon_loc, int my_rank) { + char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", + "phalf", "time"}; + int dimid[NDIM5]; + int dimid_data[NDIM4]; + char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", + "lat", "pfull", "phalf", "time"}; + int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, + NC_FLOAT, NC_FLOAT, NC_DOUBLE}; + int varid[NUM_META_VARS]; + double value_time = 2.0; + int dv; + int res; + + /* Turn off fill mode. */ + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + + /* Define dimension grid_xt. */ + if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; + + /* Define dimension grid_yt. */ + if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; + + /* Define variable grid_xt. */ + if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; + if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; + + /* Define variable lon. */ + if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; + if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); + + /* Define variable grid_yt. */ + if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; + if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; + + /* Define variable lat. */ + if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; + if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; + + /* Define dimension pfull. */ + if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; + + /* Define variable pfull and write data. */ + if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; + if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Define dimension phalf. This dim is only used by the phalf coord var. */ + if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; + + /* Define coord variable phalf and write data. */ + if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; + if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Define dimension time. */ + if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; + + /* Define variable time and write data. */ + if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; + if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + + /* In NOAA code, do all processors write the single time value? */ + if (my_rank == 0) + if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; + if (nc_redef(ncid)) ERR; + + /* Write variable grid_xt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lon data. */ + if (nc_enddef(ncid)) ERR; + /* start[0] = lon_xt_start; */ + /* start[1] = lon_yt_start; */ + /* count[0] = lon_xt_loc_size; */ + /* count[1] = lon_yt_loc_size; */ + if (nc_put_vara_double(ncid, varid[1], lon_start, lon_count, value_lon_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write grid_yt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lat data. */ + if (nc_enddef(ncid)) ERR; + /* start[0] = lat_xt_start; */ + /* start[1] = lat_yt_start; */ + /* count[0] = lat_xt_loc_size; */ + /* count[1] = lat_yt_loc_size; */ + if (nc_put_vara_double(ncid, varid[3], lat_start, lat_count, value_lat_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Specify dimensions for our data vars. */ + dimid_data[0] = dimid[4]; + dimid_data[1] = dimid[2]; + dimid_data[2] = dimid[1]; + dimid_data[3] = dimid[0]; + + /* Define data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + char data_var_name[NC_MAX_NAME + 1]; + + sprintf(data_var_name, "var_%d", dv); + if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); + else + { + res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); + } +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; + } + return 0; } @@ -81,21 +217,14 @@ main(int argc, char **argv) double data_start_time, data_stop_time; int ncid; - size_t start[NDIM4], count[NDIM4]; + size_t lat_start[NDIM2], lat_count[NDIM2]; + size_t lon_start[NDIM2], lon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; - int dimid[NDIM5]; - int dimid_data[NDIM4]; /* Variables. */ - char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", - "lat", "pfull", "phalf", "time"}; - int varid[NUM_META_VARS]; int data_varid[NUM_DATA_VARS]; - int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, - NC_FLOAT, NC_FLOAT, NC_DOUBLE}; - double value_time = 2.0; float *value_pfull_loc; size_t phalf_loc_size, phalf_start; float *value_phalf_loc; @@ -103,15 +232,12 @@ main(int argc, char **argv) double *value_grid_xt_loc; size_t grid_yt_loc_size, grid_yt_start; double *value_grid_yt_loc; - size_t lon_xt_loc_size, lon_xt_start, lon_yt_loc_size, lon_yt_start; double *value_lon_loc; - size_t lat_xt_loc_size, lat_xt_start, lat_yt_loc_size, lat_yt_start; double *value_lat_loc; float *value_data; int f; int i, j, k, dv; - int res; /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -141,29 +267,29 @@ main(int argc, char **argv) /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ /* specific to 4 pes. */ - lon_xt_loc_size = 1536; - lat_xt_loc_size = 1536; + lon_count[0] = 1536; + lat_count[0] = 1536; if (my_rank == 0 || my_rank == 2) { - lon_xt_start = 0; - lat_xt_start = 0; + lon_start[0] = 0; + lat_start[0] = 0; } else { - lon_xt_start = 1536; - lat_xt_start = 1536; + lon_start[0] = 1536; + lat_start[0] = 1536; } - lon_yt_loc_size = 768; - lat_yt_loc_size = 768; + lon_count[1] = 768; + lat_count[1] = 768; if (my_rank == 0 || my_rank == 1) { - lon_yt_start = 0; - lat_yt_start = 0; + lon_start[1] = 0; + lat_start[1] = 0; } else { - lon_yt_start = 768; - lat_yt_start = 768; + lon_start[1] = 768; + lat_start[1] = 768; } /* Allocate space on this pe to hold the coordinate var data for this pe. */ @@ -171,8 +297,8 @@ main(int argc, char **argv) if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lon_loc = malloc(lon_xt_loc_size * lon_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lat_loc = malloc(lat_xt_loc_size * lat_yt_loc_size * sizeof(double)))) ERR; + if (!(value_lon_loc = malloc(lon_count[0] * lon_count[1] * sizeof(double)))) ERR; + if (!(value_lat_loc = malloc(lat_count[0] * lat_count[1] * sizeof(double)))) ERR; /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; @@ -186,14 +312,14 @@ main(int argc, char **argv) value_grid_xt_loc[i] = my_rank * 100 + i; for (i = 0; i < grid_yt_loc_size; i++) value_grid_yt_loc[i] = my_rank * 100 + i; - for (j = 0; j < lon_yt_loc_size; j++) + for (j = 0; j < lon_count[1]; j++) { - for(i = 0; i < lon_xt_loc_size; i++) + for(i = 0; i < lon_count[0]; i++) { - value_lon_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; - value_lat_loc[j * lon_xt_loc_size + i] = my_rank * 100 + i + j; + value_lon_loc[j * lon_count[0] + i] = my_rank * 100 + i + j; + value_lat_loc[j * lon_count[0] + i] = my_rank * 100 + i + j; for (k = 0; k < data_count[1]; k++) - value_data[j * lon_xt_loc_size + i] = my_rank * 100 + i + j + k; + value_data[j * lon_count[0] + i] = my_rank * 100 + i + j + k; } } @@ -213,134 +339,10 @@ main(int argc, char **argv) meta_start_time = MPI_Wtime(); if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - if (write_metadata(ncid, dim_len, phalf_loc_size, phalf_start, value_phalf_loc)) ERR; + if (write_metadata(ncid, data_varid, s, f, dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc, + grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc, + lat_start, lat_count, value_lat_loc, lon_start, lon_count, value_lon_loc, my_rank)) ERR; - { - - /* Dimensions. */ - char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", - "phalf", "time"}; - - /* Turn off fill mode. */ - if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; - - /* Define dimension grid_xt. */ - if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; - - /* Define dimension grid_yt. */ - if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; - - /* Define variable grid_xt. */ - if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; - if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; - - /* Define variable lon. */ - if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; - if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); - - /* Define variable grid_yt. */ - if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; - if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; - - /* Define variable lat. */ - if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; - if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; - - /* Define dimension pfull. */ - if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; - - /* Define variable pfull and write data. */ - if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; - if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Define dimension phalf. This dim is only used by the phalf coord var. */ - if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; - - /* Define coord variable phalf and write data. */ - if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; - if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Define dimension time. */ - if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; - - /* Define variable time and write data. */ - if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; - if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; - if (nc_enddef(ncid)) ERR; - - /* In NOAA code, do all processors write the single time value? */ - if (my_rank == 0) - if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; - if (nc_redef(ncid)) ERR; - - /* Write variable grid_xt data. */ - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write lon data. */ - if (nc_enddef(ncid)) ERR; - start[0] = lon_xt_start; - start[1] = lon_yt_start; - count[0] = lon_xt_loc_size; - count[1] = lon_yt_loc_size; - if (nc_put_vara_double(ncid, varid[1], start, count, value_lon_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write grid_yt data. */ - if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Write lat data. */ - if (nc_enddef(ncid)) ERR; - start[0] = lat_xt_start; - start[1] = lat_yt_start; - count[0] = lat_xt_loc_size; - count[1] = lat_yt_loc_size; - if (nc_put_vara_double(ncid, varid[3], start, count, value_lat_loc)) ERR; - if (nc_redef(ncid)) ERR; - - /* Specify dimensions for our data vars. */ - dimid_data[0] = dimid[4]; - dimid_data[1] = dimid[2]; - dimid_data[2] = dimid[1]; - dimid_data[3] = dimid[0]; - - /* Define data variables. */ - for (dv = 0; dv < NUM_DATA_VARS; dv++) - { - char data_var_name[NC_MAX_NAME + 1]; - - sprintf(data_var_name, "var_%d", dv); - if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; - - /* Setting any filter only will work for HDF5-1.10.3 and later */ - /* versions. */ - if (!f) - res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); - else - { - res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); - if (!res) - res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); - } -#ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; -#else - if (res != NC_EINVAL) ERR; -#endif - - if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; - if (nc_enddef(ncid)) ERR; - } - } MPI_Barrier(MPI_COMM_WORLD); meta_stop_time = MPI_Wtime(); From fd4e388002a12c44b76e9358272edd5811ba95b5 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 14:47:59 -0600 Subject: [PATCH 29/50] added attributes --- nc_perf/tst_gfs_data_1.c | 47 +++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 1c76a348bf..7edbbd505a 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -4,7 +4,7 @@ This program tests netcdf-4 parallel I/O using the same access pattern as is used by NOAA's GFS when writing and reading model - data. + data. See https://github.com/Unidata/netcdf-fortran/issues/264 Ed Hartnett, 6/28/20 */ @@ -33,9 +33,11 @@ #define NUM_DATA_VARS 1 int -write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, - size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, size_t grid_yt_loc_size, double *value_grid_yt_loc, - size_t *lat_start, size_t *lat_count, double *value_lat_loc, size_t *lon_start, size_t *lon_count, double *value_lon_loc, int my_rank) +write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t phalf_loc_size, size_t phalf_start, + float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, + size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, + size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *lat_start, size_t *lat_count, + double *value_lat_loc, size_t *lon_start, size_t *lon_count, double *value_lon_loc, int my_rank) { char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -66,6 +68,10 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha /* Define variable lon. */ if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); + if (nc_put_att_text(ncid, varid[1], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; + if (nc_put_att_text(ncid, varid[1], "units", strlen("degrees_E"), "degrees_E")) ERR; + + if (nc_put_att_text(ncid, varid[0], "cartesian_axis", strlen("X"), "X")) ERR; /* Define variable grid_yt. */ if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; @@ -74,7 +80,11 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha /* Define variable lat. */ if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; + if (nc_put_att_text(ncid, varid[3], "long_name", strlen("T-cell latitude"), "T-cell latitude")) ERR; + if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR; + if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR; + /* Define dimension pfull. */ if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; @@ -115,10 +125,6 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha /* Write lon data. */ if (nc_enddef(ncid)) ERR; - /* start[0] = lon_xt_start; */ - /* start[1] = lon_yt_start; */ - /* count[0] = lon_xt_loc_size; */ - /* count[1] = lon_yt_loc_size; */ if (nc_put_vara_double(ncid, varid[1], lon_start, lon_count, value_lon_loc)) ERR; if (nc_redef(ncid)) ERR; @@ -129,12 +135,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha /* Write lat data. */ if (nc_enddef(ncid)) ERR; - /* start[0] = lat_xt_start; */ - /* start[1] = lat_yt_start; */ - /* count[0] = lat_xt_loc_size; */ - /* count[1] = lat_yt_loc_size; */ if (nc_put_vara_double(ncid, varid[3], lat_start, lat_count, value_lat_loc)) ERR; - if (nc_redef(ncid)) ERR; /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; @@ -148,6 +149,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha char data_var_name[NC_MAX_NAME + 1]; sprintf(data_var_name, "var_%d", dv); + if (nc_redef(ncid)) ERR; if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; /* Setting any filter only will work for HDF5-1.10.3 and later */ @@ -170,6 +172,25 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha if (nc_enddef(ncid)) ERR; } + if (nc_redef(ncid)) ERR; + if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; + if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR; + + if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR; + if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR; + if (nc_enddef(ncid)) ERR; + + if (nc_redef(ncid)) ERR; + + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + float compress_err = 42.22; + int nbits = 5; + if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR; + if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR; + } + + if (nc_enddef(ncid)) ERR; return 0; } From 5b14cc1496aa3ae2be2baa5d585352259be35999 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 14:48:10 -0600 Subject: [PATCH 30/50] added attributes --- nc_perf/tst_gfs_data_1.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 7edbbd505a..205429d006 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -21,11 +21,9 @@ #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 -#define NUM_SHUFFLE_SETTINGS 1 -/* #define NUM_SHUFFLE_SETTINGS 2 */ +#define NUM_SHUFFLE_SETTINGS 2 #ifdef HAVE_H5Z_SZIP -/* #define NUM_COMPRESSION_FILTERS 2 */ -#define NUM_COMPRESSION_FILTERS 1 +#define NUM_COMPRESSION_FILTERS 2 #else #define NUM_COMPRESSION_FILTERS 1 #endif From ab25b62892397027c4c935f401ce66b43935a937 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 1 Jul 2020 15:33:48 -0600 Subject: [PATCH 31/50] more benchmark development --- nc_perf/tst_gfs_data_1.c | 68 +++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 205429d006..a6895c5f63 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -27,11 +27,12 @@ #else #define NUM_COMPRESSION_FILTERS 1 #endif +#define NUM_DEFLATE_LEVELS 3 #define THOUSAND 1000 -#define NUM_DATA_VARS 1 +#define NUM_DATA_VARS 10 int -write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t phalf_loc_size, size_t phalf_start, +write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *lat_start, size_t *lat_count, @@ -153,7 +154,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int *dim_len, size_t pha /* Setting any filter only will work for HDF5-1.10.3 and later */ /* versions. */ if (!f) - res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, 4); + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate); else { res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); @@ -254,9 +255,10 @@ main(int argc, char **argv) double *value_lon_loc; double *value_lat_loc; float *value_data; + int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; int f; - int i, j, k, dv; + int i, j, k, dv, dl; /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -345,7 +347,7 @@ main(int argc, char **argv) if (my_rank == 0) { printf("Benchmarking creation of UFS file.\n"); - printf("comp, shuffle, meta, data\n"); + printf("comp, level, shuffle, meta, data\n"); } { int s; @@ -353,37 +355,39 @@ main(int argc, char **argv) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) { - /* nc_set_log_level(3); */ - /* Create a parallel netcdf-4 file. */ - meta_start_time = MPI_Wtime(); - if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - - if (write_metadata(ncid, data_varid, s, f, dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc, - grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc, - lat_start, lat_count, value_lat_loc, lon_start, lon_count, value_lon_loc, my_rank)) ERR; - - - MPI_Barrier(MPI_COMM_WORLD); - meta_stop_time = MPI_Wtime(); - data_start_time = MPI_Wtime(); - - /* Write one record each of the data variables. */ - for (dv = 0; dv < NUM_DATA_VARS; dv++) + for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++) { - if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)) ERR; - if (nc_redef(ncid)) ERR; - } - - /* Close the file. */ - if (nc_close(ncid)) ERR; - MPI_Barrier(MPI_COMM_WORLD); - data_stop_time = MPI_Wtime(); - if (my_rank == 0) - printf("%s, %d, %g, %g\n", (f ? "szip" : "zlib"), s, meta_stop_time - meta_start_time, data_stop_time - data_start_time); + /* nc_set_log_level(3); */ + /* Create a parallel netcdf-4 file. */ + meta_start_time = MPI_Wtime(); + if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; + + if (write_metadata(ncid, data_varid, s, f, deflate_level[dl], dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc, + grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc, + lat_start, lat_count, value_lat_loc, lon_start, lon_count, value_lon_loc, my_rank)) ERR; + + MPI_Barrier(MPI_COMM_WORLD); + meta_stop_time = MPI_Wtime(); + data_start_time = MPI_Wtime(); + + /* Write one record each of the data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)) ERR; + if (nc_redef(ncid)) ERR; + } + + /* Close the file. */ + if (nc_close(ncid)) ERR; + MPI_Barrier(MPI_COMM_WORLD); + data_stop_time = MPI_Wtime(); + if (my_rank == 0) + printf("%s, %d, %d, %g, %g\n", (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time, + data_stop_time - data_start_time); + } /* next deflate level */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ - /* free(slab_data); */ /* Free resources. */ free(value_grid_xt_loc); From b72dee72a1a9bc322484c36b1304ad87174b2807 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Thu, 2 Jul 2020 09:37:55 -0600 Subject: [PATCH 32/50] starting to add decomp for 36 tasks --- nc_perf/tst_gfs_data_1.c | 77 ++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 15 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index a6895c5f63..e18690dc1c 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -15,6 +15,7 @@ #include "err_macros.h" #include + #define FILE_NAME "tst_gfs_data_1.nc" #define NUM_META_VARS 7 #define NDIM2 2 @@ -30,7 +31,26 @@ #define NUM_DEFLATE_LEVELS 3 #define THOUSAND 1000 #define NUM_DATA_VARS 10 +#define ERR_AWFUL 1 + +/* Get the size of a file in bytes. */ +int +get_file_size(char *filename, size_t *file_size) +{ + FILE *fp; + assert(filename && file_size); + + fp = fopen(filename, "r"); + if (fp) + { + fseek(fp, 0 , SEEK_END); + *file_size = ftell(fp); + fclose(fp); + } + return 0; +} +/* Write all the metadata, including coordinate variable data. */ int write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, @@ -104,8 +124,9 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; if (nc_redef(ncid)) ERR; - /* Define dimension time. */ - if (nc_def_dim(ncid, dim_name[4], dim_len[4], &dimid[4])) ERR; + /* Define dimension time, the unlimited dimension. */ + /* if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; */ + if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; /* Define variable time and write data. */ if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; @@ -193,6 +214,8 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le return 0; } +/* Based on the MPI rank and number of tasks, calculate the + * decomposition of the 4D data. */ int decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) { @@ -205,21 +228,33 @@ decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) if (my_rank == mpi_size - 1) count[1] = count[1] + dim_len[2] % mpi_size; - if (my_rank == 0 || my_rank == 1) + if (mpi_size == 4) { - start[2] = 0; - start[3] = 0; + if (my_rank == 0 || my_rank == 1) + { + start[2] = 0; + start[3] = 0; + } + else + { + start[2] = 768; + start[3] = 768; + } + count[2] = 768; + count[3] = 1536; } - else + else if (mpi_size == 36) { - start[2] = 768; - start[3] = 768; + start[2] = my_rank * 256; + start[3] = my_rank * 512; + count[2] = 256; + count[3] = 512; } - count[2] = 768; - count[3] = 1536; - + else + return ERR_AWFUL; + printf("%d: start %ld %ld %ld %ld count %ld %ld %ld %ld\n", my_rank, start[0], - start[1], start[2], start[3], count[0], count[1], count[2], count[3]); + start[1], start[2], start[3], count[0], count[1], count[2], count[3]); return 0; } @@ -347,7 +382,7 @@ main(int argc, char **argv) if (my_rank == 0) { printf("Benchmarking creation of UFS file.\n"); - printf("comp, level, shuffle, meta, data\n"); + printf("comp, level, shuffle, meta wr time (s), data wr time (s), file size\n"); } { int s; @@ -357,6 +392,11 @@ main(int argc, char **argv) { for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++) { + size_t file_size; + + /* No deflate levels for szip. */ + if (f && dl) continue; + /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ meta_start_time = MPI_Wtime(); @@ -380,11 +420,18 @@ main(int argc, char **argv) /* Close the file. */ if (nc_close(ncid)) ERR; + + /* Stop the data timer. */ MPI_Barrier(MPI_COMM_WORLD); data_stop_time = MPI_Wtime(); + + /* Get the file size. */ + if (get_file_size(FILE_NAME, &file_size)) ERR; + + /* Print out results. */ if (my_rank == 0) - printf("%s, %d, %d, %g, %g\n", (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time, - data_stop_time - data_start_time); + printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time, + data_stop_time - data_start_time, file_size); } /* next deflate level */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ From 7a9397aed8029a57306dda91861207e51cd059b3 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Jul 2020 10:31:12 -0600 Subject: [PATCH 33/50] cleanup of 2d decomposition --- nc_perf/bm_file.c | 2 +- nc_perf/tst_gfs_data_1.c | 98 +++++++++++++++++++++++++--------------- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/nc_perf/bm_file.c b/nc_perf/bm_file.c index cfa2ab8db3..ee04cfdd7a 100644 --- a/nc_perf/bm_file.c +++ b/nc_perf/bm_file.c @@ -260,7 +260,7 @@ cmp_file(char *file1, char *file2, int *meta_read_us, size_t *data_read_us, #endif struct timeval start_time, end_time, diff_time; void *data = NULL, *data2 = NULL; - int a, v, d, dv; + int a, v, d; nc_type xtype, xtype2; int nvars, ndims, dimids[NC_MAX_VAR_DIMS], natts, real_ndims; int nvars2, ndims2, dimids2[NC_MAX_VAR_DIMS], natts2; diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index e18690dc1c..4c7c002808 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -2,9 +2,13 @@ Copyright 2020, UCAR/Unidata See COPYRIGHT file for copying and redistribution conditions. - This program tests netcdf-4 parallel I/O using the same access - pattern as is used by NOAA's GFS when writing and reading model - data. See https://github.com/Unidata/netcdf-fortran/issues/264 + This program tests and benchmarks netcdf-4 parallel I/O using the + same access pattern as is used by NOAA's GFS when writing and + reading model data. See: + https://github.com/Unidata/netcdf-fortran/issues/264. + + Also see the file gfs_sample.cdl to see what is being produced by + this program. Ed Hartnett, 6/28/20 */ @@ -22,17 +26,26 @@ #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 -#define NUM_SHUFFLE_SETTINGS 2 +/* #define NUM_SHUFFLE_SETTINGS 2 */ +#define NUM_SHUFFLE_SETTINGS 1 #ifdef HAVE_H5Z_SZIP -#define NUM_COMPRESSION_FILTERS 2 +#define NUM_COMPRESSION_FILTERS 1 +/* #define NUM_COMPRESSION_FILTERS 2 */ #else #define NUM_COMPRESSION_FILTERS 1 #endif -#define NUM_DEFLATE_LEVELS 3 +/* #define NUM_DEFLATE_LEVELS 3 */ +#define NUM_DEFLATE_LEVELS 1 #define THOUSAND 1000 #define NUM_DATA_VARS 10 #define ERR_AWFUL 1 +#define GRID_XT_LEN 3072 +#define GRID_YT_LEN 1536 +#define PFULL_LEN 127 +#define PHALF_LEN 128 +#define TIME_LEN 1 + /* Get the size of a file in bytes. */ int get_file_size(char *filename, size_t *file_size) @@ -55,8 +68,8 @@ int write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, - size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *lat_start, size_t *lat_count, - double *value_lat_loc, size_t *lon_start, size_t *lon_count, double *value_lon_loc, int my_rank) + size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, + double *value_lat_loc, double *value_lon_loc, int my_rank) { char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -145,7 +158,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le /* Write lon data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[1], lon_start, lon_count, value_lon_loc)) ERR; + if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, value_lon_loc)) ERR; if (nc_redef(ncid)) ERR; /* Write grid_yt data. */ @@ -155,7 +168,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le /* Write lat data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[3], lat_start, lat_count, value_lat_loc)) ERR; + if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, value_lat_loc)) ERR; /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; @@ -215,6 +228,14 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le } /* Based on the MPI rank and number of tasks, calculate the + * decomposition of the 2D lat/lon coordinate variables. */ +int +decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) +{ + return 0; +} + +/* Based on the MPI rank and number of tasks, calculate the * decomposition of the 4D data. */ int decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) @@ -272,11 +293,11 @@ main(int argc, char **argv) double data_start_time, data_stop_time; int ncid; - size_t lat_start[NDIM2], lat_count[NDIM2]; - size_t lon_start[NDIM2], lon_count[NDIM2]; + size_t latlon_start[NDIM2], latlon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; - int dim_len[NDIM5] = {3072, 1536, 127, 128, 1}; + int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, + TIME_LEN}; /* Variables. */ int data_varid[NUM_DATA_VARS]; @@ -290,7 +311,8 @@ main(int argc, char **argv) double *value_lon_loc; double *value_lat_loc; float *value_data; - int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; + /* int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; */ + int deflate_level[NUM_DEFLATE_LEVELS] = {1}; int f; int i, j, k, dv, dl; @@ -300,9 +322,12 @@ main(int argc, char **argv) MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - /* Determine data decomposition. */ + /* Determine 4D data decomposition to write data vars. */ if (decomp_4D(my_rank, mpi_size, dim_len, data_start, data_count)) ERR; + /* Determine 2D data decomposition to write lat/lon coordinate vars. */ + if (decomp_2D(my_rank, mpi_size, dim_len, latlon_start, latlon_count)) ERR; + /* Size of local (i.e. for this pe) grid_xt data. */ grid_xt_loc_size = dim_len[0]/mpi_size; grid_xt_start = my_rank * grid_xt_loc_size; @@ -323,29 +348,23 @@ main(int argc, char **argv) /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ /* specific to 4 pes. */ - lon_count[0] = 1536; - lat_count[0] = 1536; + latlon_count[0] = GRID_YT_LEN; if (my_rank == 0 || my_rank == 2) { - lon_start[0] = 0; - lat_start[0] = 0; + latlon_start[0] = 0; } else { - lon_start[0] = 1536; - lat_start[0] = 1536; + latlon_start[0] = GRID_YT_LEN; } - lon_count[1] = 768; - lat_count[1] = 768; + latlon_count[1] = 768; if (my_rank == 0 || my_rank == 1) { - lon_start[1] = 0; - lat_start[1] = 0; + latlon_start[1] = 0; } else { - lon_start[1] = 768; - lat_start[1] = 768; + latlon_start[1] = 768; } /* Allocate space on this pe to hold the coordinate var data for this pe. */ @@ -353,8 +372,8 @@ main(int argc, char **argv) if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lon_loc = malloc(lon_count[0] * lon_count[1] * sizeof(double)))) ERR; - if (!(value_lat_loc = malloc(lat_count[0] * lat_count[1] * sizeof(double)))) ERR; + if (!(value_lon_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (!(value_lat_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; @@ -368,14 +387,14 @@ main(int argc, char **argv) value_grid_xt_loc[i] = my_rank * 100 + i; for (i = 0; i < grid_yt_loc_size; i++) value_grid_yt_loc[i] = my_rank * 100 + i; - for (j = 0; j < lon_count[1]; j++) + for (j = 0; j < latlon_count[1]; j++) { - for(i = 0; i < lon_count[0]; i++) + for(i = 0; i < latlon_count[0]; i++) { - value_lon_loc[j * lon_count[0] + i] = my_rank * 100 + i + j; - value_lat_loc[j * lon_count[0] + i] = my_rank * 100 + i + j; + value_lon_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j; + value_lat_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j; for (k = 0; k < data_count[1]; k++) - value_data[j * lon_count[0] + i] = my_rank * 100 + i + j + k; + value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; } } @@ -404,7 +423,7 @@ main(int argc, char **argv) if (write_metadata(ncid, data_varid, s, f, deflate_level[dl], dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc, grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc, - lat_start, lat_count, value_lat_loc, lon_start, lon_count, value_lon_loc, my_rank)) ERR; + latlon_start, latlon_count, value_lat_loc, value_lon_loc, my_rank)) ERR; MPI_Barrier(MPI_COMM_WORLD); @@ -414,7 +433,14 @@ main(int argc, char **argv) /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { - if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)) ERR; + int r; + if ((r = nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data))) + { + printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl); + printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", my_rank, data_start[0], data_start[1], + data_start[2], data_start[3], data_count[0], data_count[1], data_count[2], data_count[3]); + ERR; + } if (nc_redef(ncid)) ERR; } From 86c39cf7a2eb17990995963e8eec29e224b6eff4 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sat, 4 Jul 2020 10:45:33 -0600 Subject: [PATCH 34/50] working on 2D decomposition code --- nc_perf/tst_gfs_data_1.c | 49 ++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 4c7c002808..34925013bb 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -230,8 +230,34 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le /* Based on the MPI rank and number of tasks, calculate the * decomposition of the 2D lat/lon coordinate variables. */ int -decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) +decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start, + size_t *latlon_count) { + + /* Size of local arrays (i.e. for this pe) lon and lat data. This + * is specific to 4 pes. */ + latlon_count[0] = dim_len[0]/2; + if (my_rank == 0 || my_rank == 2) + { + latlon_start[0] = 0; + } + else + { + latlon_start[0] = dim_len[0]/2; + } + latlon_count[1] = dim_len[1]/2; + if (my_rank == 0 || my_rank == 1) + { + latlon_start[1] = 0; + } + else + { + latlon_start[1] = dim_len[1]/2; + } + + printf("%d: latlon_start %ld %ld latlon_count %ld %ld\n", my_rank, latlon_start[0], + latlon_start[1], latlon_count[0], latlon_count[1]); + return 0; } @@ -346,27 +372,6 @@ main(int argc, char **argv) if (my_rank == mpi_size - 1) phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; - /* Size of local arrays (i.e. for this pe) lon and lat data. This is */ - /* specific to 4 pes. */ - latlon_count[0] = GRID_YT_LEN; - if (my_rank == 0 || my_rank == 2) - { - latlon_start[0] = 0; - } - else - { - latlon_start[0] = GRID_YT_LEN; - } - latlon_count[1] = 768; - if (my_rank == 0 || my_rank == 1) - { - latlon_start[1] = 0; - } - else - { - latlon_start[1] = 768; - } - /* Allocate space on this pe to hold the coordinate var data for this pe. */ if (!(value_pfull_loc = malloc(data_count[1] * sizeof(float)))) ERR; if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; From 4a80190c64d8324a61368c95ab4ce4b0dfbd4ad4 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 5 Jul 2020 08:54:50 -0600 Subject: [PATCH 35/50] code cleanup --- nc_perf/tst_gfs_data_1.c | 363 ++++++++++++++++++++------------------- 1 file changed, 189 insertions(+), 174 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 34925013bb..e3069b049e 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -52,38 +52,38 @@ get_file_size(char *filename, size_t *file_size) { FILE *fp; assert(filename && file_size); - + fp = fopen(filename, "r"); if (fp) { - fseek(fp, 0 , SEEK_END); - *file_size = ftell(fp); - fclose(fp); + fseek(fp, 0 , SEEK_END); + *file_size = ftell(fp); + fclose(fp); } return 0; } /* Write all the metadata, including coordinate variable data. */ int -write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, - float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, - size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, - size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, - double *value_lat_loc, double *value_lon_loc, int my_rank) +write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, + float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, + size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, + size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, + double *value_lat_loc, double *value_lon_loc, int my_rank) { char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", - "phalf", "time"}; + "phalf", "time"}; int dimid[NDIM5]; int dimid_data[NDIM4]; char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", - "lat", "pfull", "phalf", "time"}; + "lat", "pfull", "phalf", "time"}; int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, - NC_FLOAT, NC_FLOAT, NC_DOUBLE}; + NC_FLOAT, NC_FLOAT, NC_DOUBLE}; int varid[NUM_META_VARS]; double value_time = 2.0; int dv; int res; - + /* Turn off fill mode. */ if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; @@ -116,7 +116,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR; if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR; - + /* Define dimension pfull. */ if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; @@ -148,7 +148,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le /* In NOAA code, do all processors write the single time value? */ if (my_rank == 0) - if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; + if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; if (nc_redef(ncid)) ERR; /* Write variable grid_xt data. */ @@ -179,50 +179,50 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le /* Define data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { - char data_var_name[NC_MAX_NAME + 1]; - - sprintf(data_var_name, "var_%d", dv); - if (nc_redef(ncid)) ERR; - if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; - - /* Setting any filter only will work for HDF5-1.10.3 and later */ - /* versions. */ - if (!f) - res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate); - else - { - res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); - if (!res) - res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); - } + char data_var_name[NC_MAX_NAME + 1]; + + sprintf(data_var_name, "var_%d", dv); + if (nc_redef(ncid)) ERR; + if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate); + else + { + res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); + } #ifdef HDF5_SUPPORTS_PAR_FILTERS - if (res) ERR; + if (res) ERR; #else - if (res != NC_EINVAL) ERR; + if (res != NC_EINVAL) ERR; #endif - - if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; - if (nc_enddef(ncid)) ERR; + + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; } if (nc_redef(ncid)) ERR; if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR; - + if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR; if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR; if (nc_enddef(ncid)) ERR; if (nc_redef(ncid)) ERR; - + for (dv = 0; dv < NUM_DATA_VARS; dv++) { - float compress_err = 42.22; - int nbits = 5; - if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR; - if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR; + float compress_err = 42.22; + int nbits = 5; + if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR; + if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR; } - + if (nc_enddef(ncid)) ERR; return 0; } @@ -231,7 +231,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le * decomposition of the 2D lat/lon coordinate variables. */ int decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start, - size_t *latlon_count) + size_t *latlon_count) { /* Size of local arrays (i.e. for this pe) lon and lat data. This @@ -239,70 +239,70 @@ decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start, latlon_count[0] = dim_len[0]/2; if (my_rank == 0 || my_rank == 2) { - latlon_start[0] = 0; + latlon_start[0] = 0; } else { - latlon_start[0] = dim_len[0]/2; + latlon_start[0] = dim_len[0]/2; } latlon_count[1] = dim_len[1]/2; if (my_rank == 0 || my_rank == 1) { - latlon_start[1] = 0; + latlon_start[1] = 0; } else { - latlon_start[1] = dim_len[1]/2; + latlon_start[1] = dim_len[1]/2; } printf("%d: latlon_start %ld %ld latlon_count %ld %ld\n", my_rank, latlon_start[0], - latlon_start[1], latlon_count[0], latlon_count[1]); + latlon_start[1], latlon_count[0], latlon_count[1]); return 0; } -/* Based on the MPI rank and number of tasks, calculate the +/* Based on the MPI rank and number of tasks, calculate the * decomposition of the 4D data. */ int decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) { start[0] = 0; count[0] = 1; - + count[1] = dim_len[2]/mpi_size; start[1] = my_rank * count[1]; /* Add any extra to the end. */ if (my_rank == mpi_size - 1) - count[1] = count[1] + dim_len[2] % mpi_size; + count[1] = count[1] + dim_len[2] % mpi_size; if (mpi_size == 4) { - if (my_rank == 0 || my_rank == 1) - { - start[2] = 0; - start[3] = 0; - } - else - { - start[2] = 768; - start[3] = 768; - } - count[2] = 768; - count[3] = 1536; + if (my_rank == 0 || my_rank == 1) + { + start[2] = 0; + start[3] = 0; + } + else + { + start[2] = 768; + start[3] = 768; + } + count[2] = 768; + count[3] = 1536; } else if (mpi_size == 36) { - start[2] = my_rank * 256; - start[3] = my_rank * 512; - count[2] = 256; - count[3] = 512; + start[2] = my_rank * 256; + start[3] = my_rank * 512; + count[2] = 256; + count[3] = 512; } else - return ERR_AWFUL; - + return ERR_AWFUL; + printf("%d: start %ld %ld %ld %ld count %ld %ld %ld %ld\n", my_rank, start[0], - start[1], start[2], start[3], count[0], count[1], count[2], count[3]); - + start[1], start[2], start[3], count[0], count[1], count[2], count[3]); + return 0; } @@ -317,25 +317,25 @@ main(int argc, char **argv) /* For timing. */ double meta_start_time, meta_stop_time; double data_start_time, data_stop_time; - + int ncid; size_t latlon_start[NDIM2], latlon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, - TIME_LEN}; + TIME_LEN}; /* Variables. */ int data_varid[NUM_DATA_VARS]; - float *value_pfull_loc; - size_t phalf_loc_size, phalf_start; - float *value_phalf_loc; - size_t grid_xt_loc_size, grid_xt_start; - double *value_grid_xt_loc; - size_t grid_yt_loc_size, grid_yt_start; - double *value_grid_yt_loc; - double *value_lon_loc; - double *value_lat_loc; + float *pfull = NULL; + size_t phalf_size, phalf_start; + float *phalf = NULL; + size_t grid_xt_size, grid_xt_start; + double *grid_xt = NULL; + size_t grid_yt_size, grid_yt_start; + double *grid_yt = NULL; + double *lon = NULL; + double *lat = NULL; float *value_data; /* int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; */ int deflate_level[NUM_DEFLATE_LEVELS] = {1}; @@ -355,58 +355,59 @@ main(int argc, char **argv) if (decomp_2D(my_rank, mpi_size, dim_len, latlon_start, latlon_count)) ERR; /* Size of local (i.e. for this pe) grid_xt data. */ - grid_xt_loc_size = dim_len[0]/mpi_size; - grid_xt_start = my_rank * grid_xt_loc_size; + grid_xt_size = dim_len[0]/mpi_size; + grid_xt_start = my_rank * grid_xt_size; if (my_rank == mpi_size - 1) - grid_xt_loc_size = grid_xt_loc_size + dim_len[0] % mpi_size; + grid_xt_size = grid_xt_size + dim_len[0] % mpi_size; /* Size of local (i.e. for this pe) grid_yt data. */ - grid_yt_loc_size = dim_len[1]/mpi_size; - grid_yt_start = my_rank * grid_yt_loc_size; + grid_yt_size = dim_len[1]/mpi_size; + grid_yt_start = my_rank * grid_yt_size; if (my_rank == mpi_size - 1) - grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size; + grid_yt_size = grid_yt_size + dim_len[1] % mpi_size; /* Size of local (i.e. for this pe) phalf data. */ - phalf_loc_size = dim_len[3]/mpi_size; - phalf_start = my_rank * phalf_loc_size; + phalf_size = dim_len[3]/mpi_size; + phalf_start = my_rank * phalf_size; if (my_rank == mpi_size - 1) - phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size; + phalf_size = phalf_size + dim_len[3] % mpi_size; /* Allocate space on this pe to hold the coordinate var data for this pe. */ - if (!(value_pfull_loc = malloc(data_count[1] * sizeof(float)))) ERR; - if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR; - if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR; - if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR; - if (!(value_lon_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; - if (!(value_lat_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (!(pfull = malloc(data_count[1] * sizeof(float)))) ERR; + if (!(phalf = malloc(phalf_size * sizeof(float)))) ERR; + if (!(grid_xt = malloc(grid_xt_size * sizeof(double)))) ERR; + if (!(grid_yt = malloc(grid_yt_size * sizeof(double)))) ERR; + if (!(lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (!(lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; /* Some fake data for this pe to write. */ for (i = 0; i < data_count[1]; i++) - value_pfull_loc[i] = my_rank * 100 + i; - for (i = 0; i < phalf_loc_size; i++) - value_phalf_loc[i] = my_rank * 100 + i; - for (i = 0; i < grid_xt_loc_size; i++) - value_grid_xt_loc[i] = my_rank * 100 + i; - for (i = 0; i < grid_yt_loc_size; i++) - value_grid_yt_loc[i] = my_rank * 100 + i; + pfull[i] = my_rank * 100 + i; + for (i = 0; i < phalf_size; i++) + phalf[i] = my_rank * 100 + i; + for (i = 0; i < grid_xt_size; i++) + grid_xt[i] = my_rank * 100 + i; + for (i = 0; i < grid_yt_size; i++) + grid_yt[i] = my_rank * 100 + i; for (j = 0; j < latlon_count[1]; j++) { - for(i = 0; i < latlon_count[0]; i++) - { - value_lon_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j; - value_lat_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j; - for (k = 0; k < data_count[1]; k++) - value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; - } + for(i = 0; i < latlon_count[0]; i++) + { + lon[j * latlon_count[0] + i] = my_rank * 100 + i + j; + lat[j * latlon_count[0] + i] = my_rank * 100 + i + j; + for (k = 0; k < data_count[1]; k++) + value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; + } } if (my_rank == 0) { - printf("Benchmarking creation of UFS file.\n"); - printf("comp, level, shuffle, meta wr time (s), data wr time (s), file size\n"); + printf("Benchmarking creation of UFS file.\n"); + printf("comp, level, shuffle, meta wr time (s), data wr time (s), " + "file size\n"); } { int s; @@ -414,77 +415,91 @@ main(int argc, char **argv) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) { - for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++) - { - size_t file_size; - - /* No deflate levels for szip. */ - if (f && dl) continue; - - /* nc_set_log_level(3); */ - /* Create a parallel netcdf-4 file. */ - meta_start_time = MPI_Wtime(); - if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - - if (write_metadata(ncid, data_varid, s, f, deflate_level[dl], dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc, - grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc, - latlon_start, latlon_count, value_lat_loc, value_lon_loc, my_rank)) ERR; - - - MPI_Barrier(MPI_COMM_WORLD); - meta_stop_time = MPI_Wtime(); - data_start_time = MPI_Wtime(); - - /* Write one record each of the data variables. */ - for (dv = 0; dv < NUM_DATA_VARS; dv++) - { - int r; - if ((r = nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data))) - { - printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl); - printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", my_rank, data_start[0], data_start[1], - data_start[2], data_start[3], data_count[0], data_count[1], data_count[2], data_count[3]); - ERR; - } - if (nc_redef(ncid)) ERR; - } - - /* Close the file. */ - if (nc_close(ncid)) ERR; - - /* Stop the data timer. */ - MPI_Barrier(MPI_COMM_WORLD); - data_stop_time = MPI_Wtime(); - - /* Get the file size. */ - if (get_file_size(FILE_NAME, &file_size)) ERR; - - /* Print out results. */ - if (my_rank == 0) - printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time, - data_stop_time - data_start_time, file_size); - } /* next deflate level */ + for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++) + { + size_t file_size; + + /* No deflate levels for szip. */ + if (f && dl) continue; + + /* nc_set_log_level(3); */ + /* Create a parallel netcdf-4 file. */ + meta_start_time = MPI_Wtime(); + if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, + &ncid)) ERR; + + if (write_meta(ncid, data_varid, s, f, deflate_level[dl], + dim_len, phalf_size, phalf_start, phalf, + data_start, data_count, pfull, grid_xt_start, + grid_xt_size, grid_xt, grid_yt_start, + grid_yt_size, grid_yt, latlon_start, + latlon_count, lat, lon, my_rank)) ERR; + + /* Stop the timer for metadata writes. */ + MPI_Barrier(MPI_COMM_WORLD); + meta_stop_time = MPI_Wtime(); + data_start_time = MPI_Wtime(); + + /* Write one record each of the data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + int r; + if ((r = nc_put_vara_float(ncid, data_varid[dv], + data_start, data_count, value_data))) + { + printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl); + printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", + my_rank, data_start[0], data_start[1], + data_start[2], data_start[3], data_count[0], + data_count[1], data_count[2], data_count[3]); + ERR; + } + if (nc_redef(ncid)) ERR; + } + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + /* Stop the data timer. */ + MPI_Barrier(MPI_COMM_WORLD); + data_stop_time = MPI_Wtime(); + + /* Get the file size. */ + if (get_file_size(FILE_NAME, &file_size)) ERR; + + /* Print out results. */ + if (my_rank == 0) + printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"), + deflate_level[dl], s, meta_stop_time - meta_start_time, + data_stop_time - data_start_time, file_size); + } /* next deflate level */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ - /* Free resources. */ - free(value_grid_xt_loc); - free(value_grid_yt_loc); - free(value_pfull_loc); - free(value_phalf_loc); - free(value_lon_loc); - free(value_lat_loc); - free(value_data); + /* Free resources. */ + if (grid_xt) + free(grid_xt); + if (grid_yt) + free(grid_yt); + if (pfull) + free(pfull); + if (phalf) + free(phalf); + if (lon) + free(lon); + if (lat) + free(lat); + free(value_data); } if (!my_rank) SUMMARIZE_ERR; - + /* Shut down MPI. */ MPI_Finalize(); if (!my_rank) - FINAL_RESULTS; + FINAL_RESULTS; return 0; } From aa1af2f456ee6b8f24b51f024f96fa35a120b183 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 5 Jul 2020 12:23:27 -0600 Subject: [PATCH 36/50] cleaned up lat/lon code --- nc_perf/tst_gfs_data_1.c | 93 +++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index e3069b049e..d3e62ae577 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -230,29 +230,59 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s /* Based on the MPI rank and number of tasks, calculate the * decomposition of the 2D lat/lon coordinate variables. */ int -decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start, - size_t *latlon_count) +decomp_latlon(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start, + size_t *latlon_count, double **lat, double **lon) { + int i, j; - /* Size of local arrays (i.e. for this pe) lon and lat data. This - * is specific to 4 pes. */ - latlon_count[0] = dim_len[0]/2; - if (my_rank == 0 || my_rank == 2) + assert(dim_len && latlon_start && latlon_count && lat && lon && !*lat && + !*lon); + + /* Size of local arrays (i.e. for this pe) lon and lat data. */ + if (mpi_size == 1) { - latlon_start[0] = 0; + latlon_start[0] = 0; + latlon_start[1] = 0; + latlon_count[0] = dim_len[0]; + latlon_count[1] = dim_len[1]; } - else + else if (mpi_size == 4) { - latlon_start[0] = dim_len[0]/2; + latlon_count[0] = dim_len[0]/2; + if (my_rank == 0 || my_rank == 2) + { + latlon_start[0] = 0; + } + else + { + latlon_start[0] = dim_len[0]/2; + } + latlon_count[1] = dim_len[1]/2; + if (my_rank == 0 || my_rank == 1) + { + latlon_start[1] = 0; + } + else + { + latlon_start[1] = dim_len[1]/2; + } } - latlon_count[1] = dim_len[1]/2; - if (my_rank == 0 || my_rank == 1) + else if (mpi_size == 36) { - latlon_start[1] = 0; } - else + + /* Allocate storage. */ + if (!(*lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (!(*lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + + /* Now calculate some latlon values to write. */ + for (i = 0; i < latlon_count[0]; i++) { - latlon_start[1] = dim_len[1]/2; + for (j = 0; j < latlon_count[1]; j++) + { + (*lon)[j * latlon_count[0] + i] = my_rank * 100 + i + j; + (*lat)[j * latlon_count[0] + i] = my_rank * 100 + i + j; + } } printf("%d: latlon_start %ld %ld latlon_count %ld %ld\n", my_rank, latlon_start[0], @@ -275,7 +305,14 @@ decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) if (my_rank == mpi_size - 1) count[1] = count[1] + dim_len[2] % mpi_size; - if (mpi_size == 4) + if (mpi_size == 1) + { + start[2] = 0; + start[3] = 0; + count[2] = dim_len[2]; + count[3] = dim_len[3]; + } + else if (mpi_size == 4) { if (my_rank == 0 || my_rank == 1) { @@ -321,7 +358,6 @@ main(int argc, char **argv) int ncid; size_t latlon_start[NDIM2], latlon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; - int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, TIME_LEN}; @@ -352,7 +388,8 @@ main(int argc, char **argv) if (decomp_4D(my_rank, mpi_size, dim_len, data_start, data_count)) ERR; /* Determine 2D data decomposition to write lat/lon coordinate vars. */ - if (decomp_2D(my_rank, mpi_size, dim_len, latlon_start, latlon_count)) ERR; + if (decomp_latlon(my_rank, mpi_size, dim_len, latlon_start, latlon_count, + &lat, &lon)) ERR; /* Size of local (i.e. for this pe) grid_xt data. */ grid_xt_size = dim_len[0]/mpi_size; @@ -377,8 +414,6 @@ main(int argc, char **argv) if (!(phalf = malloc(phalf_size * sizeof(float)))) ERR; if (!(grid_xt = malloc(grid_xt_size * sizeof(double)))) ERR; if (!(grid_yt = malloc(grid_yt_size * sizeof(double)))) ERR; - if (!(lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; - if (!(lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; @@ -392,16 +427,16 @@ main(int argc, char **argv) grid_xt[i] = my_rank * 100 + i; for (i = 0; i < grid_yt_size; i++) grid_yt[i] = my_rank * 100 + i; - for (j = 0; j < latlon_count[1]; j++) - { - for(i = 0; i < latlon_count[0]; i++) - { - lon[j * latlon_count[0] + i] = my_rank * 100 + i + j; - lat[j * latlon_count[0] + i] = my_rank * 100 + i + j; - for (k = 0; k < data_count[1]; k++) - value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; - } - } + /* for (j = 0; j < latlon_count[1]; j++) */ + /* { */ + /* for(i = 0; i < latlon_count[0]; i++) */ + /* { */ + /* lon[j * latlon_count[0] + i] = my_rank * 100 + i + j; */ + /* lat[j * latlon_count[0] + i] = my_rank * 100 + i + j; */ + /* for (k = 0; k < data_count[1]; k++) */ + /* value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; */ + /* } */ + /* } */ if (my_rank == 0) { From 6e2a3cb3e3ca0b0ce569f2717771ae22df727ecb Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 5 Jul 2020 14:27:19 -0600 Subject: [PATCH 37/50] cleaned up grid code --- nc_perf/tst_gfs_data_1.c | 54 ++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index d3e62ae577..079e6a4192 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -343,6 +343,38 @@ decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) return 0; } +int +decomp_grid(int my_rank, int mpi_size, int *dim_len, size_t *grid_xt_start, size_t *grid_xt_size, + size_t *grid_yt_start, size_t *grid_yt_size, double **grid_xt, double **grid_yt) +{ + int i; + + /* Size of local (i.e. for this pe) grid_xt data. */ + *grid_xt_size = dim_len[0]/mpi_size; + *grid_xt_start = my_rank * *grid_xt_size; + if (my_rank == mpi_size - 1) + *grid_xt_size = *grid_xt_size + dim_len[0] % mpi_size; + + /* Size of local (i.e. for this pe) grid_yt data. */ + *grid_yt_size = dim_len[1]/mpi_size; + *grid_yt_start = my_rank * *grid_yt_size; + if (my_rank == mpi_size - 1) + *grid_yt_size = *grid_yt_size + dim_len[1] % mpi_size; + + /* Allocate storage for the grid_xy and grid_yt coordinate + * variable data. */ + if (!(*grid_xt = malloc(*grid_xt_size * sizeof(double)))) ERR; + if (!(*grid_yt = malloc(*grid_yt_size * sizeof(double)))) ERR; + + /* Fill the grid_xt and grid_yt coordinate data arrays. */ + for (i = 0; i < *grid_xt_size; i++) + (*grid_xt)[i] = my_rank * 100 + i; + for (i = 0; i < *grid_yt_size; i++) + (*grid_yt)[i] = my_rank * 100 + i; + + return 0; +} + int main(int argc, char **argv) { @@ -377,7 +409,7 @@ main(int argc, char **argv) int deflate_level[NUM_DEFLATE_LEVELS] = {1}; int f; - int i, j, k, dv, dl; + int i, dv, dl; /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -391,18 +423,8 @@ main(int argc, char **argv) if (decomp_latlon(my_rank, mpi_size, dim_len, latlon_start, latlon_count, &lat, &lon)) ERR; - /* Size of local (i.e. for this pe) grid_xt data. */ - grid_xt_size = dim_len[0]/mpi_size; - grid_xt_start = my_rank * grid_xt_size; - if (my_rank == mpi_size - 1) - grid_xt_size = grid_xt_size + dim_len[0] % mpi_size; - - /* Size of local (i.e. for this pe) grid_yt data. */ - grid_yt_size = dim_len[1]/mpi_size; - grid_yt_start = my_rank * grid_yt_size; - if (my_rank == mpi_size - 1) - grid_yt_size = grid_yt_size + dim_len[1] % mpi_size; - + if (decomp_grid(my_rank, mpi_size, dim_len, &grid_xt_start, &grid_xt_size, &grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR; + /* Size of local (i.e. for this pe) phalf data. */ phalf_size = dim_len[3]/mpi_size; phalf_start = my_rank * phalf_size; @@ -412,8 +434,6 @@ main(int argc, char **argv) /* Allocate space on this pe to hold the coordinate var data for this pe. */ if (!(pfull = malloc(data_count[1] * sizeof(float)))) ERR; if (!(phalf = malloc(phalf_size * sizeof(float)))) ERR; - if (!(grid_xt = malloc(grid_xt_size * sizeof(double)))) ERR; - if (!(grid_yt = malloc(grid_yt_size * sizeof(double)))) ERR; /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; @@ -423,10 +443,6 @@ main(int argc, char **argv) pfull[i] = my_rank * 100 + i; for (i = 0; i < phalf_size; i++) phalf[i] = my_rank * 100 + i; - for (i = 0; i < grid_xt_size; i++) - grid_xt[i] = my_rank * 100 + i; - for (i = 0; i < grid_yt_size; i++) - grid_yt[i] = my_rank * 100 + i; /* for (j = 0; j < latlon_count[1]; j++) */ /* { */ /* for(i = 0; i < latlon_count[0]; i++) */ From a9b5c33ba75b14d969c3aeb67b169370fe096650 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 5 Jul 2020 15:33:26 -0600 Subject: [PATCH 38/50] dealing with coord var cleanup --- nc_perf/tst_gfs_data_1.c | 67 ++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 079e6a4192..ce4254b5e0 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -343,6 +343,8 @@ decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count) return 0; } +/* Decompose the grid_xt and grid_yt coordinate vars, and also come up + * with some data. */ int decomp_grid(int my_rank, int mpi_size, int *dim_len, size_t *grid_xt_start, size_t *grid_xt_size, size_t *grid_yt_start, size_t *grid_yt_size, double **grid_xt, double **grid_yt) @@ -375,6 +377,32 @@ decomp_grid(int my_rank, int mpi_size, int *dim_len, size_t *grid_xt_start, size return 0; } +int +decomp_p(int my_rank, int mpi_size, size_t *data_count, int *dim_len, + size_t *phalf_start, size_t *phalf_size, float **phalf, float **pfull) +{ + int i; + + /* Size of local (i.e. for this pe) phalf data. */ + *phalf_size = dim_len[3]/mpi_size; + *phalf_start = my_rank * *phalf_size; + if (my_rank == mpi_size - 1) + *phalf_size = *phalf_size + dim_len[3] % mpi_size; + + /* Allocate space on this pe to hold the coordinate var data for this pe. */ + if (!(*pfull = malloc(data_count[1] * sizeof(float)))) ERR; + if (!(*phalf = malloc(*phalf_size * sizeof(float)))) ERR; + + /* Some fake data for this pe to write. */ + for (i = 0; i < data_count[1]; i++) + (*pfull)[i] = my_rank * 100 + i; + for (i = 0; i < *phalf_size; i++) + (*phalf)[i] = my_rank * 100 + i; + + return 0; +} + + int main(int argc, char **argv) { @@ -409,7 +437,7 @@ main(int argc, char **argv) int deflate_level[NUM_DEFLATE_LEVELS] = {1}; int f; - int i, dv, dl; + int i, j, k, dv, dl; /* Initialize MPI. */ MPI_Init(&argc, &argv); @@ -423,36 +451,21 @@ main(int argc, char **argv) if (decomp_latlon(my_rank, mpi_size, dim_len, latlon_start, latlon_count, &lat, &lon)) ERR; - if (decomp_grid(my_rank, mpi_size, dim_len, &grid_xt_start, &grid_xt_size, &grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR; - - /* Size of local (i.e. for this pe) phalf data. */ - phalf_size = dim_len[3]/mpi_size; - phalf_start = my_rank * phalf_size; - if (my_rank == mpi_size - 1) - phalf_size = phalf_size + dim_len[3] % mpi_size; - - /* Allocate space on this pe to hold the coordinate var data for this pe. */ - if (!(pfull = malloc(data_count[1] * sizeof(float)))) ERR; - if (!(phalf = malloc(phalf_size * sizeof(float)))) ERR; + /* Decompose grid_xt and grid_yt coordiate vars. */ + if (decomp_grid(my_rank, mpi_size, dim_len, &grid_xt_start, &grid_xt_size, + &grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR; + /* Decompose phalf and pfull. */ + if (decomp_p(my_rank, mpi_size, data_count, dim_len, &phalf_start, &phalf_size, &phalf, + &pfull)) ERR; + /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; - /* Some fake data for this pe to write. */ - for (i = 0; i < data_count[1]; i++) - pfull[i] = my_rank * 100 + i; - for (i = 0; i < phalf_size; i++) - phalf[i] = my_rank * 100 + i; - /* for (j = 0; j < latlon_count[1]; j++) */ - /* { */ - /* for(i = 0; i < latlon_count[0]; i++) */ - /* { */ - /* lon[j * latlon_count[0] + i] = my_rank * 100 + i + j; */ - /* lat[j * latlon_count[0] + i] = my_rank * 100 + i + j; */ - /* for (k = 0; k < data_count[1]; k++) */ - /* value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k; */ - /* } */ - /* } */ + for (k = 0; k < data_count[1]; k++) + for (j = 0; j < data_count[2]; j++) + for(i = 0; i < data_count[3]; i++) + value_data[j * data_count[3] + i] = my_rank * 100 + i + j + k; if (my_rank == 0) { From 8faa7ed9461055d498773b5974883de680b11e19 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Sun, 5 Jul 2020 15:57:07 -0600 Subject: [PATCH 39/50] added new metadata writing function --- nc_perf/tst_gfs_data_1.c | 252 ++++++++++++++++++++++++++++++++------- 1 file changed, 211 insertions(+), 41 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index ce4254b5e0..235ec9c5c7 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -22,6 +22,7 @@ #define FILE_NAME "tst_gfs_data_1.nc" #define NUM_META_VARS 7 +#define NUM_META_TRIES 2 #define NDIM2 2 #define NDIM4 4 #define NDIM5 5 @@ -227,6 +228,170 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s return 0; } +/* Write all the metadata, including coordinate variable data. */ +int +write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, + float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, + size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, + size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, + double *value_lat_loc, double *value_lon_loc, int my_rank) +{ + char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", + "phalf", "time"}; + int dimid[NDIM5]; + int dimid_data[NDIM4]; + char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", + "lat", "pfull", "phalf", "time"}; + int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, + NC_FLOAT, NC_FLOAT, NC_DOUBLE}; + int varid[NUM_META_VARS]; + double value_time = 2.0; + int dv; + int res; + + /* Turn off fill mode. */ + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + + /* Define dimension grid_xt. */ + if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; + + /* Define dimension grid_yt. */ + if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; + + /* Define variable grid_xt. */ + if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; + if (nc_var_par_access(ncid, varid[0], NC_INDEPENDENT)) ERR; + + /* Define variable lon. */ + if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; + if (nc_var_par_access(ncid, varid[1], NC_INDEPENDENT)); + if (nc_put_att_text(ncid, varid[1], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; + if (nc_put_att_text(ncid, varid[1], "units", strlen("degrees_E"), "degrees_E")) ERR; + + if (nc_put_att_text(ncid, varid[0], "cartesian_axis", strlen("X"), "X")) ERR; + + /* Define variable grid_yt. */ + if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; + if (nc_var_par_access(ncid, varid[2], NC_INDEPENDENT)) ERR; + + /* Define variable lat. */ + if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; + if (nc_var_par_access(ncid, varid[3], NC_INDEPENDENT)) ERR; + if (nc_put_att_text(ncid, varid[3], "long_name", strlen("T-cell latitude"), "T-cell latitude")) ERR; + if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR; + + if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR; + + /* Define dimension pfull. */ + if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; + + /* Define variable pfull and write data. */ + if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; + if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Define dimension phalf. This dim is only used by the phalf coord var. */ + if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; + + /* Define coord variable phalf and write data. */ + if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; + if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Define dimension time, the unlimited dimension. */ + /* if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; */ + if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; + + /* Define variable time and write data. */ + if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; + if (nc_var_par_access(ncid, varid[6], NC_INDEPENDENT)) ERR; + if (nc_enddef(ncid)) ERR; + + /* In NOAA code, do all processors write the single time value? */ + if (my_rank == 0) + if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; + if (nc_redef(ncid)) ERR; + + /* Write variable grid_xt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lon data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, value_lon_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write grid_yt data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_redef(ncid)) ERR; + + /* Write lat data. */ + if (nc_enddef(ncid)) ERR; + if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, value_lat_loc)) ERR; + + /* Specify dimensions for our data vars. */ + dimid_data[0] = dimid[4]; + dimid_data[1] = dimid[2]; + dimid_data[2] = dimid[1]; + dimid_data[3] = dimid[0]; + + /* Define data variables. */ + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + char data_var_name[NC_MAX_NAME + 1]; + + sprintf(data_var_name, "var_%d", dv); + if (nc_redef(ncid)) ERR; + if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later */ + /* versions. */ + if (!f) + res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate); + else + { + res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, data_varid[dv], 32, 32); + } +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR; + if (nc_enddef(ncid)) ERR; + } + + if (nc_redef(ncid)) ERR; + if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; + if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR; + + if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR; + if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR; + if (nc_enddef(ncid)) ERR; + + if (nc_redef(ncid)) ERR; + + for (dv = 0; dv < NUM_DATA_VARS; dv++) + { + float compress_err = 42.22; + int nbits = 5; + if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR; + if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR; + } + + if (nc_enddef(ncid)) ERR; + return 0; +} + /* Based on the MPI rank and number of tasks, calculate the * decomposition of the 2D lat/lon coordinate variables. */ int @@ -436,7 +601,7 @@ main(int argc, char **argv) /* int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; */ int deflate_level[NUM_DEFLATE_LEVELS] = {1}; - int f; + int f, s, meta; int i, j, k, dv, dl; /* Initialize MPI. */ @@ -456,12 +621,14 @@ main(int argc, char **argv) &grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR; /* Decompose phalf and pfull. */ - if (decomp_p(my_rank, mpi_size, data_count, dim_len, &phalf_start, &phalf_size, &phalf, - &pfull)) ERR; + if (decomp_p(my_rank, mpi_size, data_count, dim_len, &phalf_start, + &phalf_size, &phalf, &pfull)) ERR; /* Allocate space to hold the data. */ - if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; + if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * + sizeof(float)))) ERR; + /* Create some data. */ for (k = 0; k < data_count[1]; k++) for (j = 0; j < data_count[2]; j++) for(i = 0; i < data_count[3]; i++) @@ -470,11 +637,11 @@ main(int argc, char **argv) if (my_rank == 0) { printf("Benchmarking creation of UFS file.\n"); - printf("comp, level, shuffle, meta wr time (s), data wr time (s), " + printf("meta, comp, level, shuffle, meta wr time (s), data wr time (s), " "file size\n"); } + for (meta = 0; meta < NUM_META_TRIES; meta++) { - int s; for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) @@ -492,12 +659,24 @@ main(int argc, char **argv) if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; - if (write_meta(ncid, data_varid, s, f, deflate_level[dl], - dim_len, phalf_size, phalf_start, phalf, - data_start, data_count, pfull, grid_xt_start, - grid_xt_size, grid_xt, grid_yt_start, - grid_yt_size, grid_yt, latlon_start, - latlon_count, lat, lon, my_rank)) ERR; + if (meta) + { + if (write_meta2(ncid, data_varid, s, f, deflate_level[dl], + dim_len, phalf_size, phalf_start, phalf, + data_start, data_count, pfull, grid_xt_start, + grid_xt_size, grid_xt, grid_yt_start, + grid_yt_size, grid_yt, latlon_start, + latlon_count, lat, lon, my_rank)) ERR; + } + else + { + if (write_meta(ncid, data_varid, s, f, deflate_level[dl], + dim_len, phalf_size, phalf_start, phalf, + data_start, data_count, pfull, grid_xt_start, + grid_xt_size, grid_xt, grid_yt_start, + grid_yt_size, grid_yt, latlon_start, + latlon_count, lat, lon, my_rank)) ERR; + } /* Stop the timer for metadata writes. */ MPI_Barrier(MPI_COMM_WORLD); @@ -507,18 +686,9 @@ main(int argc, char **argv) /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { - int r; - if ((r = nc_put_vara_float(ncid, data_varid[dv], - data_start, data_count, value_data))) - { - printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl); - printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", - my_rank, data_start[0], data_start[1], - data_start[2], data_start[3], data_count[0], - data_count[1], data_count[2], data_count[3]); - ERR; - } - if (nc_redef(ncid)) ERR; + if (nc_put_vara_float(ncid, data_varid[dv], data_start, + data_count, value_data)) ERR; + if (nc_redef(ncid)) ERR; } /* Close the file. */ @@ -533,28 +703,28 @@ main(int argc, char **argv) /* Print out results. */ if (my_rank == 0) - printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"), + printf("%d, %s, %d, %d, %g, %g, %ld\n", meta, (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time, data_stop_time - data_start_time, file_size); } /* next deflate level */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ - - /* Free resources. */ - if (grid_xt) - free(grid_xt); - if (grid_yt) - free(grid_yt); - if (pfull) - free(pfull); - if (phalf) - free(phalf); - if (lon) - free(lon); - if (lat) - free(lat); - free(value_data); - } + } /* next ed */ + + /* Free resources. */ + if (grid_xt) + free(grid_xt); + if (grid_yt) + free(grid_yt); + if (pfull) + free(pfull); + if (phalf) + free(phalf); + if (lon) + free(lon); + if (lat) + free(lat); + free(value_data); if (!my_rank) SUMMARIZE_ERR; From 7b477784f553e38549be102ae8064a9ae6d1342f Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 6 Jul 2020 07:53:11 -0600 Subject: [PATCH 40/50] starting to check metadata --- nc_perf/tst_gfs_data_1.c | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 235ec9c5c7..0fc172ccd3 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -64,13 +64,24 @@ get_file_size(char *filename, size_t *file_size) return 0; } +/* Check all the metadata, including coordinate variable data. */ +int +check_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, + float *phalf, size_t *data_start, size_t *data_count, float *pfull, + size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, + size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, + double *lat, double *lon, int my_rank) +{ + return 0; +} + /* Write all the metadata, including coordinate variable data. */ int -write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, - float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, - size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, - size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, - double *value_lat_loc, double *value_lon_loc, int my_rank) +write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, + float *phalf, size_t *data_start, size_t *data_count, float *pfull, + size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, + size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, + double *lat, double *lon, int my_rank) { char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -125,7 +136,7 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; + if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], pfull)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension phalf. This dim is only used by the phalf coord var. */ @@ -135,7 +146,7 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_size, phalf)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension time, the unlimited dimension. */ @@ -154,22 +165,22 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s /* Write variable grid_xt data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_size, grid_xt)) ERR; if (nc_redef(ncid)) ERR; /* Write lon data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, value_lon_loc)) ERR; + if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, lon)) ERR; if (nc_redef(ncid)) ERR; /* Write grid_yt data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_size, grid_yt)) ERR; if (nc_redef(ncid)) ERR; /* Write lat data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, value_lat_loc)) ERR; + if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, lat)) ERR; /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; @@ -230,11 +241,11 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s /* Write all the metadata, including coordinate variable data. */ int -write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start, - float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc, - size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start, - size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count, - double *value_lat_loc, double *value_lon_loc, int my_rank) +write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, + float *phalf, size_t *data_start, size_t *data_count, float *pfull, + size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, + size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, + double *lat, double *lon, int my_rank) { char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; @@ -289,7 +300,7 @@ write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_var_par_access(ncid, varid[4], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], value_pfull_loc)) ERR; + if (nc_put_vara_float(ncid, varid[4], &data_start[1], &data_count[1], pfull)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension phalf. This dim is only used by the phalf coord var. */ @@ -299,7 +310,7 @@ write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; if (nc_var_par_access(ncid, varid[5], NC_INDEPENDENT)) ERR; if (nc_enddef(ncid)) ERR; - if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_loc_size, value_phalf_loc)) ERR; + if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_size, phalf)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension time, the unlimited dimension. */ @@ -318,22 +329,22 @@ write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, /* Write variable grid_xt data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_loc_size, value_grid_xt_loc)) ERR; + if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_size, grid_xt)) ERR; if (nc_redef(ncid)) ERR; /* Write lon data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, value_lon_loc)) ERR; + if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, lon)) ERR; if (nc_redef(ncid)) ERR; /* Write grid_yt data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_loc_size, value_grid_yt_loc)) ERR; + if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_size, grid_yt)) ERR; if (nc_redef(ncid)) ERR; /* Write lat data. */ if (nc_enddef(ncid)) ERR; - if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, value_lat_loc)) ERR; + if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, lat)) ERR; /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; @@ -701,6 +712,16 @@ main(int argc, char **argv) /* Get the file size. */ if (get_file_size(FILE_NAME, &file_size)) ERR; + /* Check the file for correctness. */ + if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; + if (check_meta(ncid, data_varid, s, f, deflate_level[dl], + dim_len, phalf_size, phalf_start, phalf, + data_start, data_count, pfull, grid_xt_start, + grid_xt_size, grid_xt, grid_yt_start, + grid_yt_size, grid_yt, latlon_start, + latlon_count, lat, lon, my_rank)) ERR; + if (nc_close(ncid)) ERR; + /* Print out results. */ if (my_rank == 0) printf("%d, %s, %d, %d, %g, %g, %ld\n", meta, (f ? "szip" : "zlib"), From 661cde52035a05876e057135205ac5c64cae3dfc Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 6 Jul 2020 08:14:15 -0600 Subject: [PATCH 41/50] more checking metadata --- nc_perf/tst_gfs_data_1.c | 74 ++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 0fc172ccd3..915c56898b 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -47,6 +47,15 @@ #define PHALF_LEN 128 #define TIME_LEN 1 +char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", + "phalf", "time"}; +char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", + "lat", "pfull", "phalf", "time"}; +int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, + NC_FLOAT, NC_FLOAT, NC_DOUBLE}; +int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, + TIME_LEN}; + /* Get the size of a file in bytes. */ int get_file_size(char *filename, size_t *file_size) @@ -66,31 +75,54 @@ get_file_size(char *filename, size_t *file_size) /* Check all the metadata, including coordinate variable data. */ int -check_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, - float *phalf, size_t *data_start, size_t *data_count, float *pfull, - size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, - size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, - double *lat, double *lon, int my_rank) +check_meta(int ncid, int *data_varid, int s, int f, int deflate, + size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, + size_t *data_count, float *pfull, size_t grid_xt_start, + size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, + size_t grid_yt_size, double *grid_yt, size_t *latlon_start, + size_t *latlon_count, double *lat, double *lon, int my_rank) { + int ndims, nvars, natts, unlimdimid; + char name_in[NC_MAX_NAME + 1]; + int xtype_in; + int ndims_in; + int dimids_in[NDIM4]; + size_t len_in; + int d, v; + + /* Check number of dims, vars, atts. */ + if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; + if (ndims != NDIM5 || nvars != NUM_META_VARS + NUM_DATA_VARS || + natts != 0 || unlimdimid != 4) ERR; + + /* Check the dimensions. */ + for (d = 0; d < NDIM5; d++) + { + if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR; + if (strcmp(name_in, dim_name[d]) || len_in != dim_len[d]) ERR; + } + + /* Check metadata vars. */ + for (v = 0; v < NUM_META_VARS; v++) + { + if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, + &natts)) ERR; + if (strcmp(name_in, var_name[v])) ERR; + } + return 0; } /* Write all the metadata, including coordinate variable data. */ int -write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, +write_meta(int ncid, int *data_varid, int s, int f, int deflate, size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, size_t *data_count, float *pfull, size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, double *lat, double *lon, int my_rank) { - char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", - "phalf", "time"}; int dimid[NDIM5]; int dimid_data[NDIM4]; - char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", - "lat", "pfull", "phalf", "time"}; - int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, - NC_FLOAT, NC_FLOAT, NC_DOUBLE}; int varid[NUM_META_VARS]; double value_time = 2.0; int dv; @@ -150,8 +182,8 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s if (nc_redef(ncid)) ERR; /* Define dimension time, the unlimited dimension. */ - /* if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; */ - if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; + if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; + /* if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; */ /* Define variable time and write data. */ if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; @@ -241,7 +273,7 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, s /* Write all the metadata, including coordinate variable data. */ int -write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_size, size_t phalf_start, +write_meta2(int ncid, int *data_varid, int s, int f, int deflate, size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, size_t *data_count, float *pfull, size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, @@ -314,8 +346,8 @@ write_meta2(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, if (nc_redef(ncid)) ERR; /* Define dimension time, the unlimited dimension. */ - /* if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; */ - if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; + if (nc_def_dim(ncid, dim_name[4], NC_UNLIMITED, &dimid[4])) ERR; + /* if (nc_def_dim(ncid, dim_name[4], 1, &dimid[4])) ERR; */ /* Define variable time and write data. */ if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; @@ -594,8 +626,6 @@ main(int argc, char **argv) int ncid; size_t latlon_start[NDIM2], latlon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; - int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, - TIME_LEN}; /* Variables. */ int data_varid[NUM_DATA_VARS]; @@ -673,7 +703,7 @@ main(int argc, char **argv) if (meta) { if (write_meta2(ncid, data_varid, s, f, deflate_level[dl], - dim_len, phalf_size, phalf_start, phalf, + phalf_size, phalf_start, phalf, data_start, data_count, pfull, grid_xt_start, grid_xt_size, grid_xt, grid_yt_start, grid_yt_size, grid_yt, latlon_start, @@ -682,7 +712,7 @@ main(int argc, char **argv) else { if (write_meta(ncid, data_varid, s, f, deflate_level[dl], - dim_len, phalf_size, phalf_start, phalf, + phalf_size, phalf_start, phalf, data_start, data_count, pfull, grid_xt_start, grid_xt_size, grid_xt, grid_yt_start, grid_yt_size, grid_yt, latlon_start, @@ -715,7 +745,7 @@ main(int argc, char **argv) /* Check the file for correctness. */ if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; if (check_meta(ncid, data_varid, s, f, deflate_level[dl], - dim_len, phalf_size, phalf_start, phalf, + phalf_size, phalf_start, phalf, data_start, data_count, pfull, grid_xt_start, grid_xt_size, grid_xt, grid_yt_start, grid_yt_size, grid_yt, latlon_start, From 6b2820ca95995569498c7850c88b3f0e46df899c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 6 Jul 2020 08:26:35 -0600 Subject: [PATCH 42/50] now checking some coord vars --- nc_perf/tst_gfs_data_1.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 915c56898b..feabe3ce08 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -88,7 +88,9 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, int ndims_in; int dimids_in[NDIM4]; size_t len_in; - int d, v; + double *grid_xt_in, *grid_yt_in; + float *phalf_in, *pfull_in; + int d, v, i; /* Check number of dims, vars, atts. */ if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; @@ -107,8 +109,40 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, { if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, &natts)) ERR; - if (strcmp(name_in, var_name[v])) ERR; + if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v]) ERR; } + + /* Check the values for grid_xt. */ + if (!(grid_xt_in = malloc(grid_xt_size * sizeof(double)))) ERR; + if (nc_get_var_double(ncid, 0, grid_xt_in)) ERR; + for (i = 0; i < grid_xt_size; i++) + if (grid_xt_in[i] != grid_xt[i]) ERR; + free(grid_xt_in); + + /* Check the values for lon. */ + + /* Check the values for grid_yt. */ + if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR; + if (nc_get_var_double(ncid, 2, grid_yt_in)) ERR; + for (i = 0; i < grid_yt_size; i++) + if (grid_yt_in[i] != grid_yt[i]) ERR; + free(grid_yt_in); + + /* Check the values for lat. */ + + /* Check the values for pfull. */ + if (!(pfull_in = malloc(data_count[1] * sizeof(float)))) ERR; + if (nc_get_var_float(ncid, 4, pfull_in)) ERR; + for (i = 0; i < data_count[1]; i++) + if (pfull_in[i] != pfull[i]) ERR; + free(pfull_in); + + /* Check the values for phalf. */ + if (!(phalf_in = malloc(phalf_size * sizeof(float)))) ERR; + if (nc_get_var_float(ncid, 5, phalf_in)) ERR; + for (i = 0; i < phalf_size; i++) + if (phalf_in[i] != phalf[i]) ERR; + free(phalf_in); return 0; } From f9e753495e82fc0043c967409461cca835f02b2f Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 08:04:06 -0600 Subject: [PATCH 43/50] more checking of metadata and coord vars --- nc_perf/tst_gfs_data_1.c | 1 + 1 file changed, 1 insertion(+) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index feabe3ce08..e56d639405 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -89,6 +89,7 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, int dimids_in[NDIM4]; size_t len_in; double *grid_xt_in, *grid_yt_in; + double *lat_in, lon_in; float *phalf_in, *pfull_in; int d, v, i; From a13c0a2d2355e03277b0d5243e8345a1969711d4 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 08:49:12 -0600 Subject: [PATCH 44/50] starting to use get_vara functions to check metadata --- nc_perf/tst_gfs_data_1.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index e56d639405..19f7641c62 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -115,12 +115,13 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, /* Check the values for grid_xt. */ if (!(grid_xt_in = malloc(grid_xt_size * sizeof(double)))) ERR; - if (nc_get_var_double(ncid, 0, grid_xt_in)) ERR; + if (nc_get_vara_double(ncid, 0, &grid_xt_start, &grid_xt_size, grid_xt_in)) ERR; for (i = 0; i < grid_xt_size; i++) if (grid_xt_in[i] != grid_xt[i]) ERR; free(grid_xt_in); /* Check the values for lon. */ + /* Check the values for grid_yt. */ if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR; From d13035b01e1b5b13f8abd7d04bd0f2677249546c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 08:58:38 -0600 Subject: [PATCH 45/50] further checking of metadata --- nc_perf/tst_gfs_data_1.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 19f7641c62..a4ecb4b4ec 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -125,7 +125,7 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, /* Check the values for grid_yt. */ if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR; - if (nc_get_var_double(ncid, 2, grid_yt_in)) ERR; + if (nc_get_vara_double(ncid, 2, &grid_yt_start, &grid_yt_size, grid_yt_in)) ERR; for (i = 0; i < grid_yt_size; i++) if (grid_yt_in[i] != grid_yt[i]) ERR; free(grid_yt_in); @@ -134,14 +134,14 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, /* Check the values for pfull. */ if (!(pfull_in = malloc(data_count[1] * sizeof(float)))) ERR; - if (nc_get_var_float(ncid, 4, pfull_in)) ERR; + if (nc_get_vara_float(ncid, 4, &data_start[1], &data_count[1], pfull_in)) ERR; for (i = 0; i < data_count[1]; i++) if (pfull_in[i] != pfull[i]) ERR; free(pfull_in); /* Check the values for phalf. */ if (!(phalf_in = malloc(phalf_size * sizeof(float)))) ERR; - if (nc_get_var_float(ncid, 5, phalf_in)) ERR; + if (nc_get_vara_float(ncid, 5, &phalf_start, &phalf_size, phalf_in)) ERR; for (i = 0; i < phalf_size; i++) if (phalf_in[i] != phalf[i]) ERR; free(phalf_in); @@ -151,11 +151,12 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, /* Write all the metadata, including coordinate variable data. */ int -write_meta(int ncid, int *data_varid, int s, int f, int deflate, size_t phalf_size, size_t phalf_start, - float *phalf, size_t *data_start, size_t *data_count, float *pfull, - size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, - size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, - double *lat, double *lon, int my_rank) +write_meta(int ncid, int *data_varid, int s, int f, int deflate, + size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, + size_t *data_count, float *pfull, size_t grid_xt_start, + size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, + size_t grid_yt_size, double *grid_yt, size_t *latlon_start, + size_t *latlon_count, double *lat, double *lon, int my_rank) { int dimid[NDIM5]; int dimid_data[NDIM4]; From d539e03c687adf65675d5bf1559d14fc78b9b03c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 09:01:38 -0600 Subject: [PATCH 46/50] further checking of metadata --- nc_perf/tst_gfs_data_1.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index a4ecb4b4ec..fcb4c20c88 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -121,7 +121,11 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, free(grid_xt_in); /* Check the values for lon. */ - + if (!(lon_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lon_in)) ERR; + /* for (i = 0; i < grid_yt_size; i++) */ + /* if (grid_yt_in[i] != grid_yt[i]) ERR; */ + free(lon_in); /* Check the values for grid_yt. */ if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR; From 2dad8d1bffa63f30a1e5f2c49b0ca2706683ed45 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 09:04:07 -0600 Subject: [PATCH 47/50] now checking lat/lon coord values --- nc_perf/tst_gfs_data_1.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index fcb4c20c88..91eae84308 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -89,7 +89,7 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, int dimids_in[NDIM4]; size_t len_in; double *grid_xt_in, *grid_yt_in; - double *lat_in, lon_in; + double *lat_in, *lon_in; float *phalf_in, *pfull_in; int d, v, i; @@ -123,8 +123,8 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, /* Check the values for lon. */ if (!(lon_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lon_in)) ERR; - /* for (i = 0; i < grid_yt_size; i++) */ - /* if (grid_yt_in[i] != grid_yt[i]) ERR; */ + for (i = 0; i < latlon_count[0] * latlon_count[1]; i++) + if (lon_in[i] != lon[i]) ERR; free(lon_in); /* Check the values for grid_yt. */ @@ -135,6 +135,11 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, free(grid_yt_in); /* Check the values for lat. */ + if (!(lat_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; + if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lat_in)) ERR; + for (i = 0; i < latlon_count[0] * latlon_count[1]; i++) + if (lat_in[i] != lat[i]) ERR; + free(lat_in); /* Check the values for pfull. */ if (!(pfull_in = malloc(data_count[1] * sizeof(float)))) ERR; From 257155268679d6bc1cf777a1a3608f32d56bb4fa Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 11:25:49 -0600 Subject: [PATCH 48/50] changed output to be more readable --- nc_perf/tst_gfs_data_1.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/nc_perf/tst_gfs_data_1.c b/nc_perf/tst_gfs_data_1.c index 91eae84308..58f0fe35b8 100644 --- a/nc_perf/tst_gfs_data_1.c +++ b/nc_perf/tst_gfs_data_1.c @@ -27,16 +27,13 @@ #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 -/* #define NUM_SHUFFLE_SETTINGS 2 */ -#define NUM_SHUFFLE_SETTINGS 1 +#define NUM_SHUFFLE_SETTINGS 2 #ifdef HAVE_H5Z_SZIP -#define NUM_COMPRESSION_FILTERS 1 -/* #define NUM_COMPRESSION_FILTERS 2 */ +#define NUM_COMPRESSION_FILTERS 2 #else #define NUM_COMPRESSION_FILTERS 1 #endif -/* #define NUM_DEFLATE_LEVELS 3 */ -#define NUM_DEFLATE_LEVELS 1 +#define NUM_DEFLATE_LEVELS 3 #define THOUSAND 1000 #define NUM_DATA_VARS 10 #define ERR_AWFUL 1 @@ -685,8 +682,7 @@ main(int argc, char **argv) double *lon = NULL; double *lat = NULL; float *value_data; - /* int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; */ - int deflate_level[NUM_DEFLATE_LEVELS] = {1}; + int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; int f, s, meta; int i, j, k, dv, dl; @@ -724,8 +720,8 @@ main(int argc, char **argv) if (my_rank == 0) { printf("Benchmarking creation of UFS file.\n"); - printf("meta, comp, level, shuffle, meta wr time (s), data wr time (s), " - "file size\n"); + printf("meta, comp, level, shuffle, meta wr time (s), data wr rate (MB/s), " + "file size (MB)\n"); } for (meta = 0; meta < NUM_META_TRIES; meta++) { @@ -788,7 +784,7 @@ main(int argc, char **argv) /* Get the file size. */ if (get_file_size(FILE_NAME, &file_size)) ERR; - /* Check the file for correctness. */ + /* Check the file metadata for correctness. */ if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; if (check_meta(ncid, data_varid, s, f, deflate_level[dl], phalf_size, phalf_start, phalf, @@ -800,9 +796,17 @@ main(int argc, char **argv) /* Print out results. */ if (my_rank == 0) - printf("%d, %s, %d, %d, %g, %g, %ld\n", meta, (f ? "szip" : "zlib"), - deflate_level[dl], s, meta_stop_time - meta_start_time, - data_stop_time - data_start_time, file_size); + { + float data_size, data_rate; + data_size = NUM_DATA_VARS * dim_len[0] * dim_len[1] * + dim_len[3] * sizeof(float)/1000000; + data_rate = data_size / (data_stop_time - data_start_time); + printf("%d, %s, %d, %d, %g, %g, %g\n", meta, + (f ? "szip" : "zlib"), deflate_level[dl], s, + meta_stop_time - meta_start_time, data_rate, + (float)file_size/1000000); + } + MPI_Barrier(MPI_COMM_WORLD); } /* next deflate level */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ From ab5a1c496c5b44ac6a92a9e980403d30e47b30b8 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 11:36:22 -0600 Subject: [PATCH 49/50] updated RELEASE_NOTES --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 48a8b2fcb4..7da536059f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD +* [Enhancement] Added new benchmark program nc_perf/tst_gfs_data_1.c, built only when --enable-benchmarks is in configure. See [https://github.com/Unidata/netcdf-c/pull/1777]. * [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774]. * [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in]. * [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763]. From 94c285afed6b8482ebea34e4d74603f834607e28 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Tue, 7 Jul 2020 11:38:37 -0600 Subject: [PATCH 50/50] updated RELEASE_NOTES --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 7da536059f..27d81f50e6 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD -* [Enhancement] Added new benchmark program nc_perf/tst_gfs_data_1.c, built only when --enable-benchmarks is in configure. See [https://github.com/Unidata/netcdf-c/pull/1777]. +* [Enhancement] Added new parallel I/O benchmark program to mimic NOAA UFS data writes, built when --enable-benchmarks is in configure. See [https://github.com/Unidata/netcdf-c/pull/1777]. * [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774]. * [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in]. * [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].