Merge pull request #7 from PlasmaFAIR/feature/compression

Add `compression` argument
PlasmaFAIR · Apr 4, 2023 · 1b60c9a · 1b60c9a
2 parents af71a7c + 7ef02f9
commit 1b60c9a
Show file tree

Hide file tree

Showing 4 changed files with 211 additions and 22 deletions.
diff --git a/src/neasyf.f90 b/src/neasyf.f90
@@ -67,6 +67,18 @@ module neasyf
   private
   public :: neasyf_open, neasyf_close, neasyf_type, neasyf_dim
   public :: neasyf_write, neasyf_read, neasyf_error, neasyf_metadata
+  public :: neasyf_default_compression
+
+  !> Default compression level to use when creating variables. The default is
+  !> zero, no compression. Non-zero values should be between 1-9
+  !>
+  !> This can be overridden explicitly in calls to [[neasyf_write]].
+  !>
+  !> Setting this to a non-zero value also enables the `shuffle` filter. There
+  !> is some discussion of how compression works in netCDF in the [documentation
+  !> for the C
+  !> library](https://docs.unidata.ucar.edu/netcdf-c/current/group__variables.html#ga59dad3301f241a7eb86f31b339af2d26)
+  integer :: neasyf_default_compression = 0
 
   integer, parameter :: nf_kind = kind(NF90_INT)
 
@@ -759,15 +771,15 @@ end function polymorphic_get_var_rank7
   !> Create a dimension if it doesn't already exist.
   !>
   !> If the dimension doesn't exist, also create a variable of the same name and
-  !> fill it with [[values]], or the integers in the range `1..dim_size`. The
-  !> optional argument [[unlimited]] can be used to make this dimension
+  !> fill it with `values`, or the integers in the range `1..dim_size`. The
+  !> optional argument `unlimited` can be used to make this dimension
   !> unlimited in extent.
   !>
   !> Optional arguments "unit" and "long_name" allow you to create attributes
   !> of the same names.
   !>
   !> The netCDF IDs of the dimension and corresponding variable can be returned
-  !> through [[dimid]] and [[varid]] respectively.
+  !> through `dimid` and `varid` respectively.
   subroutine neasyf_dim(parent_id, name, values, dim_size, dimid, varid, units, long_name, unlimited)
 
     use netcdf, only : nf90_inq_dimid, nf90_inq_varid, nf90_def_var, nf90_def_dim, nf90_put_var, nf90_put_att, &
@@ -990,7 +1002,7 @@ end subroutine neasyf_write_scalar
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank1(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1011,12 +1023,19 @@ subroutine neasyf_write_rank1(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1037,9 +1056,15 @@ subroutine neasyf_write_rank1(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1087,7 +1112,7 @@ end subroutine neasyf_write_rank1
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank2(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1108,12 +1133,19 @@ subroutine neasyf_write_rank2(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1134,9 +1166,15 @@ subroutine neasyf_write_rank2(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1184,7 +1222,7 @@ end subroutine neasyf_write_rank2
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank3(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1205,12 +1243,19 @@ subroutine neasyf_write_rank3(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1231,9 +1276,15 @@ subroutine neasyf_write_rank3(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1281,7 +1332,7 @@ end subroutine neasyf_write_rank3
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank4(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1302,12 +1353,19 @@ subroutine neasyf_write_rank4(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1328,9 +1386,15 @@ subroutine neasyf_write_rank4(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1378,7 +1442,7 @@ end subroutine neasyf_write_rank4
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank5(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1399,12 +1463,19 @@ subroutine neasyf_write_rank5(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1425,9 +1496,15 @@ subroutine neasyf_write_rank5(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1475,7 +1552,7 @@ end subroutine neasyf_write_rank5
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank6(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1496,12 +1573,19 @@ subroutine neasyf_write_rank6(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1522,9 +1606,15 @@ subroutine neasyf_write_rank6(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then
@@ -1572,7 +1662,7 @@ end subroutine neasyf_write_rank6
   !>
   !> which avoids the need to manually pad each dimension name with spaces.
   subroutine neasyf_write_rank7(parent_id, name, values, dim_ids, dim_names, &
-       varid, units, long_name, start, count, stride, map)
+       varid, units, long_name, start, count, stride, map, compression)
     use, intrinsic :: iso_fortran_env, only : error_unit
     use netcdf, only : nf90_inq_varid, nf90_def_var, nf90_put_var, nf90_put_att, &
          nf90_inq_dimid, NF90_NOERR, NF90_ENOTVAR, NF90_EDIMMETA
@@ -1593,12 +1683,19 @@ subroutine neasyf_write_rank7(parent_id, name, values, dim_ids, dim_names, &
     !> Long descriptive name
     character(len=*), optional, intent(in) :: long_name
     integer, dimension(:), optional, intent(in) :: start, count, stride, map
+    !> If non-zero, use compression.
+    !>
+    !> Enables the `shuffle` netCDF filter and sets the `deflate_level`
+    !> parameter to `compression`. You can set the default compression through
+    !> [[neasyf_default_compression]]
+    integer, optional, intent(in) :: compression
 
     integer, dimension(:), allocatable :: local_dim_ids
     integer :: dim_index
     integer(nf_kind) :: nf_type
     integer :: status
     integer :: var_id
+    integer :: local_compression
 
     status = nf90_inq_varid(parent_id, name, var_id)
     ! Variable doesn't exist, so let's create it
@@ -1619,9 +1716,15 @@ subroutine neasyf_write_rank7(parent_id, name, values, dim_ids, dim_names, &
         end do
       end if
 
+      local_compression = neasyf_default_compression
+      if (present(compression)) then
+        local_compression = compression
+      end if
+
       nf_type = neasyf_type(values)
       ! TODO: check if nf_type indicates a derived type
-      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id)
+      status = nf90_def_var(parent_id, name, nf_type, local_dim_ids, var_id, &
+           shuffle=(local_compression > 0), deflate_level=local_compression)
       deallocate(local_dim_ids)
 
       if (present(units)) then

diff --git a/src/neasyf.in.f90 b/src/neasyf.in.f90
@@ -67,6 +67,18 @@ module neasyf
   private
   public :: neasyf_open, neasyf_close, neasyf_type, neasyf_dim
   public :: neasyf_write, neasyf_read, neasyf_error, neasyf_metadata
+  public :: neasyf_default_compression
+
+  !> Default compression level to use when creating variables. The default is
+  !> zero, no compression. Non-zero values should be between 1-9
+  !>
+  !> This can be overridden explicitly in calls to [[neasyf_write]].
+  !>
+  !> Setting this to a non-zero value also enables the `shuffle` filter. There
+  !> is some discussion of how compression works in netCDF in the [documentation
+  !> for the C
+  !> library](https://docs.unidata.ucar.edu/netcdf-c/current/group__variables.html#ga59dad3301f241a7eb86f31b339af2d26)
+  integer :: neasyf_default_compression = 0
 
   integer, parameter :: nf_kind = kind(NF90_INT)
 
@@ -305,15 +317,15 @@ end function polymorphic_get_var_scalar
   !> Create a dimension if it doesn't already exist.
   !>
   !> If the dimension doesn't exist, also create a variable of the same name and
-  !> fill it with [[values]], or the integers in the range `1..dim_size`. The
-  !> optional argument [[unlimited]] can be used to make this dimension
+  !> fill it with `values`, or the integers in the range `1..dim_size`. The
+  !> optional argument `unlimited` can be used to make this dimension
   !> unlimited in extent.
   !>
   !> Optional arguments "unit" and "long_name" allow you to create attributes
   !> of the same names.
   !>
   !> The netCDF IDs of the dimension and corresponding variable can be returned
-  !> through [[dimid]] and [[varid]] respectively.
+  !> through `dimid` and `varid` respectively.
   subroutine neasyf_dim(parent_id, name, values, dim_size, dimid, varid, units, long_name, unlimited)
 
     use netcdf, only : nf90_inq_dimid, nf90_inq_varid, nf90_def_var, nf90_def_dim, nf90_put_var, nf90_put_att, &