Skip to content

Commit

Permalink
move MODIS fetch functions to Fetcher (not tested)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yujie-W
Yujie-W committed Dec 4, 2021
1 parent d72cc39 commit e454113
Show file tree
Hide file tree
Showing 17 changed files with 165 additions and 284 deletions.
9 changes: 1 addition & 8 deletions Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,27 @@
name = "GriddingMachine"
uuid = "f20cf718-bf4d-4727-bc8f-485b1f283ac6"
authors = ["Yujie Wang"]
version = "0.2.2"
version = "0.2.3"

[deps]
Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
PkgUtility = "0d262f2c-28e9-492c-8e19-d7a5c4f11611"
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"

[compat]
Conda = "1.5.2"
DataFrames = "1.2.2"
DocStringExtensions = "0.8.5"
Glob = "1.3.0"
HTTP = "0.9.14"
JSON = "0.21.2"
PkgUtility = "0.1.13"
ProgressMeter = "1.7.1"
PyCall = "1.92.3"
UnPack = "1.0.2"
julia = "1.6"
14 changes: 2 additions & 12 deletions docs/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,16 +128,11 @@ version = "0.4.2"
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"

[[deps.Glob]]
git-tree-sha1 = "4df9f7e06108728ebf00a0a11edee4b29a482bb2"
uuid = "c27321d9-0574-5035-807b-f59d2c89b15c"
version = "1.3.0"

[[deps.GriddingMachine]]
deps = ["Artifacts", "Conda", "DataFrames", "Dates", "Distributed", "DocStringExtensions", "Glob", "HTTP", "JSON", "LazyArtifacts", "PkgUtility", "ProgressMeter", "PyCall", "Test", "UnPack"]
deps = ["Artifacts", "Conda", "Dates", "DocStringExtensions", "HTTP", "JSON", "LazyArtifacts", "PkgUtility", "ProgressMeter", "PyCall", "Test"]
path = ".."
uuid = "f20cf718-bf4d-4727-bc8f-485b1f283ac6"
version = "0.2.2"
version = "0.2.3"

[[deps.HDF5_jll]]
deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"]
Expand Down Expand Up @@ -444,11 +439,6 @@ version = "1.3.0"
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[deps.UnPack]]
git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
version = "1.0.2"

[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

Expand Down
1 change: 1 addition & 0 deletions docs/src/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ Fetcher.ERA5LandHourly
Fetcher.ERA5SingleLevelsHourly
Fetcher.install_cdsapi!
Fetcher.update_CDSAPI_info!
Fetcher.update_MODIS_password!
Fetcher.fetch_data!
Fetcher.fetch_data!(dt::Fetcher.ERA5LandHourly, year::Int)
Fetcher.fetch_data!(dt::Fetcher.ERA5SingleLevelsHourly, year::Int)
Expand Down
160 changes: 147 additions & 13 deletions src/Fetcher.jl
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
module Fetcher

using Conda: add
using DocStringExtensions: METHODLIST
using Dates: isleapyear
using DocStringExtensions: TYPEDEF, METHODLIST
using PkgUtility: month_days
using ProgressMeter: @showprogress
using PyCall: pyimport


Expand All @@ -18,6 +20,10 @@ export fetch_data!
CDSAPI_PORTAL = "https://cds.climate.copernicus.eu/api/v2";
CDSAPI_KEY = "";
CDSAPI_CLIENT = nothing;
MODIS_HOME = "/net/fluo/data1/data/MODIS";
MODIS_PORTAL = "https://e4ftl01.cr.usgs.gov";
MODIS_USER_ID = "";
MODIS_USER_PWD = "";


# constants
Expand Down Expand Up @@ -363,21 +369,46 @@ const ERA5_SINGLE_LEVELS_SELECTION =[


"""
struct ERA5LandHourly
$(TYPEDEF)
ERA5 Land hourly data from 1981 to present: cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-land
"""
struct ERA5LandHourly end;


"""
struct ERA5SingleLevelsHourly
$(TYPEDEF)
ERA5 Single Level hourly data from 1981 to present: cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels
"""
struct ERA5SingleLevelsHourly end;


"""
$(TYPEDEF)
Hierachy of AbstractMODIS500m
- [`MOD15A2Hv006LAI`](@ref)
"""
abstract type AbstractMODIS500m end


"""
$(TYPEDEF)
Terra Surface Reflectance
"""
struct MOD09A1v006NIRv <: AbstractMODIS500m end


"""
$(TYPEDEF)
Leaf area index
"""
struct MOD15A2Hv006LAI <: AbstractMODIS500m end


"""
install_cdsapi!()
Expand Down Expand Up @@ -407,9 +438,9 @@ function update_CDSAPI_info!()
end;

# update password and client
cdsapi = pyimport("cdsapi");
if isfile("$(homedir())/.cdsapirc")
@info "Load cdsapi information from $(homedir())/.cdsapirc";
cdsapi = pyimport("cdsapi");
CDSAPI_CLIENT = cdsapi.Client();
CDSAPI_KEY = CDSAPI_CLIENT.key;
CDSAPI_PORTAL = CDSAPI_CLIENT.url;
Expand All @@ -419,7 +450,6 @@ function update_CDSAPI_info!()
CDSAPI_PORTAL = readline();
@info "Please indicate the cdsapi key combo (uid:api-key):";
CDSAPI_KEY = read(Base.getpass("Password (invisble)"), String);
cdsapi = pyimport("cdsapi");
CDSAPI_CLIENT = cdsapi.Client(url=CDSAPI_PORTAL, key=CDSAPI_KEY);
end;
end;
Expand All @@ -429,7 +459,27 @@ end;


"""
Fetch data from the server. Supported methods are
update_MODIS_password!()
Update global user name and password for LP DAAC, if either of them is empty
"""
function update_MODIS_password!()
# input MODIS_USER_ID and MODIS_USER_PWD
global MODIS_USER_ID, MODIS_USER_PWD;
if MODIS_USER_ID == "" || MODIS_USER_PWD == ""
@warn "Do not share your user name and password with others.";
@info "Please indicate your user name for LP DAAC data portal:";
MODIS_USER_ID = readline();
@info "Please indicate your password for LP DAAC Data portal:";
MODIS_USER_PWD = read(Base.getpass("Password (invisble)"), String);
end;

return nothing
end;


"""
Fetch data from the server. Supported methods are
$(METHODLIST)
Expand Down Expand Up @@ -463,10 +513,10 @@ fetch_data!(dt::ERA5LandHourly, year::Int; vars::Array{String,1} = ERA5_LAND_SEL
@info "Querying $(_item)";

# file name to save the downloaded dataset
file_name = joinpath(folder, "$(_item)_LAND_$(year).nc");
_file_name = joinpath(folder, "$(_item)_LAND_$(year).nc");

# retrieve data if file does not exist
if !isfile(file_name)
if !isfile(_file_name)
CDSAPI_CLIENT.retrieve(
"reanalysis-era5-land",
Dict(
Expand All @@ -477,7 +527,7 @@ fetch_data!(dt::ERA5LandHourly, year::Int; vars::Array{String,1} = ERA5_LAND_SEL
"day" => ["$(i)" for i in 1:31],
"time" => ERA5_LAND_TIMES,
),
file_name
_file_name
);
end;
end;
Expand Down Expand Up @@ -512,11 +562,10 @@ fetch_data!(dt::ERA5SingleLevelsHourly, year::Int; vars::Array{String,1} = ERA5_
@info "Querying $(_item)";

# file name to save the downloaded dataset
file_name = joinpath(folder, "$(_item)_SL_$(year).nc");
@show file_name;
_file_name = joinpath(folder, "$(_item)_SL_$(year).nc");

# retrieve data if file does not exist
if !isfile(file_name)
if !isfile(_file_name)
CDSAPI_CLIENT.retrieve(
"reanalysis-era5-single-levels",
Dict(
Expand All @@ -528,7 +577,7 @@ fetch_data!(dt::ERA5SingleLevelsHourly, year::Int; vars::Array{String,1} = ERA5_
"day" => ["$(i)" for i in 1:31],
"time" => ERA5_LAND_TIMES,
),
file_name
_file_name
);
end;
end;
Expand All @@ -537,4 +586,89 @@ fetch_data!(dt::ERA5SingleLevelsHourly, year::Int; vars::Array{String,1} = ERA5_
);


"""
fetch_data!(data_url::String, data_loc::String, year::Int, label::String)
Download raw product data from MODIS, given
- `data_url` URL of the data
- `data_loc` Where to save the downloaded data
- `year` Which year of MODIS data to download
- `label` Label in the MODIS data that help the identification
"""
fetch_data!(data_url::String, data_loc::String, year::Int, label::String) =(
# number of days per year
_nday = isleapyear(year) ? 366 : 365;

# fetch file list in each folder
@info "Fetching file name to download...";
_list_urls = [];
_list_locs = [];
for _doy in 1:8:_nday
#folder = parse_date(year, doy, ".") * "/";
_folder = "/";
try
@info "Fetching file list from $(data_url * _folder)";
download(data_url * _folder, "temp.html");
for _line in readlines("temp.html")
if contains(_line, ".hdf\">")
_ini = findfirst(label, _line);
_end = findfirst(".hdf", _line);
_nam = _line[_ini[1]:_end[1]+3];
push!(_list_urls, data_url * _folder * _nam);
push!(_list_locs, data_loc * string(year) * "/" * _nam);
end;
end;
rm("temp.html");
catch err
@info "Unable to fetch files from $(_folder), skip it...";
end;
end;

# download files if the file does not exist
@info "Downloading files...";
@showprogress for _i in eachindex(_list_locs)
_url = _list_urls[_i];
_loc = _list_locs[_i];
if Sys.which("wget") !== nothing
if !isfile(_loc)
_lst = `-q $(_url) -O $(_loc)`;
_psd = `--user $(MODIS_USER_ID) --password $(MODIS_USER_PWD)`;
run(`wget $(_psd) $(_lst)`);
end;
else
@warn "wget not found, exit the loop";
break
end;
end;

return nothing
);


"""
"""
fetch_data!(dt::MOD15A2Hv006LAI, year::Int) = (
update_MODIS_password!();

data_url = "$(MODIS_PORTAL)/MOLT/MOD15A2H.006/";
data_loc = "$(MODIS_HOME)/MOD15A2H.006/original/";

fetch_data!(data_url, data_loc, year, "MOD15A2H");

return nothing
);


fetch_data!(dt::MOD09A1v006NIRv, year::Int) = (
update_MODIS_password!();

data_url = "$(MODIS_PORTAL)/MOLT/MOD09A1.006/";
data_loc = "$(MODIS_HOME)/MOD09A1.006/original/";

fetch_data!(data_url, data_loc, year, "MOD09A1");

return nothing
);


end # module
Loading

0 comments on commit e454113

Please sign in to comment.