Skip to content

Commit

Permalink
allow to use a different masking value than missing
Browse files Browse the repository at this point in the history
  • Loading branch information
Alexander-Barth committed Feb 6, 2024
1 parent 9c7e442 commit dcc130f
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 8 deletions.
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
GRIB = "b16dfd50-4035-11e9-28d4-9dfe17e6779b"

[compat]
CommonDataModel = "^0.2.1, 0.3"
CommonDataModel = "0.3.4"
DataStructures = "0.18"
DiskArrays = "0.3"
GRIB = "0.3, 0.4"
Expand Down
3 changes: 2 additions & 1 deletion src/GRIBDatasets.jl
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ using GRIB
using DataStructures
import DiskArrays as DA
using CommonDataModel: AbstractDataset, AbstractVariable, show_dim, CFVariable
import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib, dataset
import CommonDataModel: path, name, dimnames, isopen, attribnames, attrib,
dataset, maskingvalue
import CommonDataModel as CDM

const DEFAULT_EPOCH = DateTime(1970, 1, 1, 0, 0)
Expand Down
3 changes: 2 additions & 1 deletion src/cfvariables.jl
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@

_get_dim(cfvar::CFVariable, dimname) = _get_dim(cfvar.var, dimname)
function cfvariable(ds, varname)
function cfvariable(ds, varname; maskingvalue = maskingvalue(ds))
v = Variable(ds, string(varname))
misval = missing_value(v)
CDM.cfvariable(
ds, varname;
_v = v,
missing_value = isnothing(misval) ? eltype(v)[] : [misval],
attrib = cflayer_attributes(v),
maskingvalue = maskingvalue,
)
end

Expand Down
11 changes: 7 additions & 4 deletions src/dataset.jl
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,21 @@ It can be created with the path to the GRIB file:
ds = GRIBDataset(example_file);
```
"""
struct GRIBDataset{T, N} <: AbstractDataset
struct GRIBDataset{T, N, Tmaskingvalue} <: AbstractDataset
index::FileIndex{T}
dims::NTuple{N, AbstractDim}
attrib::Dict{String, Any}
maskingvalue::Tmaskingvalue
end

const Dataset = GRIBDataset

function GRIBDataset(index::FileIndex)
GRIBDataset(index, _alldims(index), dataset_attributes(index))
function GRIBDataset(index::FileIndex; maskingvalue = missing)
GRIBDataset(index, _alldims(index), dataset_attributes(index), maskingvalue)
end

GRIBDataset(filepath::AbstractString; filter_by_values = Dict()) = GRIBDataset(FileIndex(filepath; filter_by_values))
GRIBDataset(filepath::AbstractString; filter_by_values = Dict(), kwargs...) =
GRIBDataset(FileIndex(filepath; filter_by_values); kwargs...)

Base.keys(ds::Dataset) = getvars(ds)
Base.haskey(ds::Dataset, key) = key in keys(ds)
Expand All @@ -46,6 +48,7 @@ dimnames(ds::GRIBDataset) = keys(ds.dims)

attribnames(ds::GRIBDataset) = keys(ds.attrib)
attrib(ds::GRIBDataset, attribname::String) = ds.attrib[attribname]
maskingvalue(ds::GRIBDataset) = ds.maskingvalue

# _dim_values(ds::GRIBDataset, dim::Dimension{Horizontal}) = _dim_values(ds.index, dim)

Expand Down
16 changes: 15 additions & 1 deletion test/dataset.jl
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ using GRIBDatasets: CDM
grib_path = joinpath(dir_testfiles, "era5-levels-members.grib")
ds = GRIBDataset(grib_path)
dsmis = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib"))
dsNaN = GRIBDataset(joinpath(dir_testfiles, "fields_with_missing_values.grib"),maskingvalue = NaN)
index = ds.index

varstring = "z"
Expand Down Expand Up @@ -87,12 +88,25 @@ using GRIBDatasets: CDM
@testset "cfvariable and missing" begin
cfvar = cfvariable(ds, varstring)
cfvarmis = cfvariable(dsmis, "t2m")
cfvarNaN = cfvariable(dsNaN, "t2m")
A = cfvar[:,:,1,1,1]
Amis = cfvarmis[:,:,1,1]
ANaN = cfvarNaN[:,:,1,1]

# With CommonDataModel, we necessarily get a Union{Missing, Float64}, even if there's no missing.
@test_broken eltype(A) == Float64
@test eltype(Amis) == Union{Missing, Float64}
@test eltype(ANaN) == Float64

# test the use of a different maskingvalue per dataset
@test eltype(dsmis["t2m"]) == Union{Missing,Float64}
@test eltype(dsNaN["t2m"]) == Float64
@test ismissing(Amis[1,1,1])
@test isnan(ANaN[1,1,1])

# test the use of a different maskingvalue per variable
A2NaN = cfvariable(dsmis,"t2m", maskingvalue = NaN)
@test isnan(A2NaN[1,1,1])
end

@testset "cfvariable coordinate" begin
Expand Down Expand Up @@ -181,4 +195,4 @@ end
@time ds = GRIBDataset(testfile)
end

end
end

0 comments on commit dcc130f

Please sign in to comment.