From ae375b8c6a77b0a2cd75492ee3e8782d8e3d204b Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Sat, 30 Mar 2024 16:24:05 +0700 Subject: [PATCH] Search for groups of granules and convert them. --- src/ICESat-2/ICESat-2.jl | 7 +++-- src/granule.jl | 4 +-- src/search.jl | 56 ++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/ICESat-2/ICESat-2.jl b/src/ICESat-2/ICESat-2.jl index 5e1e582..fcf6b25 100644 --- a/src/ICESat-2/ICESat-2.jl +++ b/src/ICESat-2/ICESat-2.jl @@ -129,8 +129,8 @@ Converts the granule `g` to the product `product`, by guessing the correct name. """ function Base.convert(product::Symbol, g::ICESat2_Granule{T}) where {T} g = ICESat2_Granule{product}( - replace(replace(g.id, String(T) => String(product)), lowercase(String(T)) => lowercase(String(product))), - replace(replace(g.url, String(T) => String(product)), lowercase(String(T)) => lowercase(String(product))), + _convert(g.id, T, product), + _convert(g.url, T, product), g.info, g.polygons, ) @@ -146,6 +146,9 @@ function Base.convert(product::Symbol, g::ICESat2_Granule{T}) where {T} g end +function _convert(s::AbstractString, old::Symbol, new::Symbol) + replace(replace(s, String(old) => String(new)), lowercase(String(old)) => lowercase(String(new))) +end """ info(g::ICESat2_Granule) diff --git a/src/granule.jl b/src/granule.jl index 9f55525..fa038a4 100644 --- a/src/granule.jl +++ b/src/granule.jl @@ -217,10 +217,10 @@ function _sync!(granules, folder, all; kwargs...) isempty(granules) && error("No granules found in provided folder(s).") g = first(granules) ngranules = if length(granules) == 0 || !haskey(info(granules[end]), :date) || all - Set(search(g; kwargs...)) + Set(search(mission(g), sproduct(g); kwargs...)) else sort!(granules, by = x -> x.id) - Set(search(g; after = info(granules[end]).date, kwargs...)) + Set(search(mission(g), sproduct(g); after = info(granules[end]).date, kwargs...)) end setdiff!(ngranules, Set(granules)) download!(collect(ngranules), folder) diff --git a/src/search.jl b/src/search.jl index e426424..19f874e 100644 --- a/src/search.jl +++ b/src/search.jl @@ -28,7 +28,7 @@ function search( version::Int = 2, before::Union{Nothing,DateTime} = nothing, after::Union{Nothing,DateTime} = nothing, - id::Union{Nothing,String}=nothing, + id::Union{Nothing,String,Vector{String}} = nothing, provider::String = "LPDAAC_ECS", )::Vector{GEDI_Granule} startswith(string(product), prefix(m)) || throw(ArgumentError("Wrong product $product for $(mission(m)) mission.")) @@ -38,7 +38,15 @@ function search( end granules = - earthdata_search(short_name = string(product), bounding_box = extent, version = version, provider = provider, before = before, after = after, id = id) + earthdata_search( + short_name = string(product), + bounding_box = extent, + version = version, + provider = provider, + before = before, + after = after, + id = id, + ) length(granules) == 0 && @warn "No granules found, did you specify the correct parameters, such as version?" filter!(g -> !isnothing(g.https_url), granules) map( @@ -59,7 +67,7 @@ function search( version::Int = 6, before::Union{Nothing,DateTime} = nothing, after::Union{Nothing,DateTime} = nothing, - id::Union{Nothing,String}=nothing, + id::Union{Nothing,String,Vector{String}} = nothing, s3::Bool = false, provider::String = s3 ? "NSIDC_CPRD" : "NSIDC_ECS", )::Vector{ICESat2_Granule} @@ -70,7 +78,15 @@ function search( end granules = - earthdata_search(short_name = string(product), bounding_box = extent, version = version, provider = provider, before = before, after = after, id = id) + earthdata_search( + short_name = string(product), + bounding_box = extent, + version = version, + provider = provider, + before = before, + after = after, + id = id, + ) length(granules) == 0 && @warn "No granules found, did you specify the correct parameters, such as version?" s3 ? filter!(g -> !isnothing(g.s3_url), granules) : filter!(g -> !isnothing(g.https_url), granules) map( @@ -91,7 +107,7 @@ function search( version::Int = 34, before::Union{Nothing,DateTime} = nothing, after::Union{Nothing,DateTime} = nothing, - id::Union{Nothing,String}=nothing, + id::Union{Nothing,String,Vector{String}} = nothing, s3::Bool = false, provider::String = s3 ? "NSIDC_CPRD" : "NSIDC_ECS", )::Vector{ICESat_Granule} @@ -102,7 +118,15 @@ function search( end granules = - earthdata_search(short_name = string(product), bounding_box = extent, version = version, provider = provider, before = before, after = after, id = id) + earthdata_search( + short_name = string(product), + bounding_box = extent, + version = version, + provider = provider, + before = before, + after = after, + id = id, + ) length(granules) == 0 && @warn "No granules found, did you specify the correct parameters, such as version?" s3 ? filter!(g -> !isnothing(g.s3_url), granules) : filter!(g -> !isnothing(g.https_url), granules) map( @@ -141,10 +165,26 @@ function search(mission::Symbol, product::Symbol, args...; kwargs...) end function search(g::Granule; kwargs...) - initial = (; version = info(g).version) + initial = (; version = info(g).version, id = g.id) + only(search(mission(g), sproduct(g); merge(initial, kwargs)...)) +end + +function search(gg::Vector{<:Granule}; kwargs...) + g = first(gg) + initial = (; version = info(g).version, id = map(x -> x.id, gg)) search(mission(g), sproduct(g); merge(initial, kwargs)...) end +function search(g::Granule, product::Symbol; kwargs...) + g = SpaceLiDAR.convert(product, g) + search(g, kwargs...) +end + +function search(gg::Vector{<:Granule}, product::Symbol; kwargs...) + gg = SpaceLiDAR.convert.(product, gg) + search(gg, kwargs...) +end + function parse_polygon(polygons, T = Float64) o = Vector{Vector{Vector{Vector{T}}}}() for polygon in polygons @@ -217,7 +257,7 @@ function earthdata_search(; provider::Union{Nothing,String} = "NSIDC_CPRD", before::Union{Nothing,DateTime} = nothing, after::Union{Nothing,DateTime} = nothing, - id::Union{Nothing,String} = nothing, + id::Union{Nothing,String,Vector{String}} = nothing, all_pages::Bool = true, page_size = 2000, page_num = 1, # unused