Skip to content

Commit

Permalink
Merge #67
Browse files Browse the repository at this point in the history 
67: up deps of PkgUtility and NetcdfIO r=Yujie-W a=Yujie-W



Co-authored-by: Yujie Wang <[email protected]>
  • Loading branch information
@bors @Yujie-W
bors[bot] and Yujie-W authored Feb 3, 2022
2 parents ec6fdc9 + eea1321 commit 6aa4847
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 39 deletions.
6 changes: 3 additions & 3 deletions Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "GriddingMachine"
uuid = "f20cf718-bf4d-4727-bc8f-485b1f283ac6"
authors = ["Yujie Wang"]
version = "0.2.4"
version = "0.2.5"

[deps]
Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
Expand All @@ -22,8 +22,8 @@ Conda = "1.5.2"
DocStringExtensions = "0.8.5"
HTTP = "0.9.14"
JSON = "0.21.2"
NetcdfIO = "0.1.2"
PkgUtility = "0.1.13"
NetcdfIO = "0.1.2, 0.2"
PkgUtility = "0.1.13, 0.2.0"
ProgressMeter = "1.7.1"
PyCall = "1.92.3"
julia = "1.6"
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ julia> Pkg.add("GriddingMachine");
GriddingMachine has the following sub-modules, some of which are in development. The sub-modules are
| Sub-module | Functionality | Ready to use |
|:------------|:--------------------------------|:-------------|
| Blender | Regrid the gridded datasets | No |
| Blender | Regrid the gridded datasets | Testing |
| Collector | Distribute the gridded datasets | v0.2 |
| Fetcher | Download ungridded datasets | No |
| Fetcher | Download ungridded datasets | Testing |
| Indexer | Read the gridded datasets | v0.2 |
| Partitioner | Sort the ungridded datasets | No |
| Partitioner | Sort the ungridded datasets | Testing |
| Requestor | Request gridded datasets | v0.2 |

See [`API`][gm-api] for more detailed information about how to use [`GriddingMachine.jl`][gm-url].
Expand Down
80 changes: 55 additions & 25 deletions docs/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,34 @@ version = "0.1.1"

[[deps.CLIMAParameters]]
deps = ["Test"]
git-tree-sha1 = "1d2f657dbc634c69a58d71c619bbf7f51005a0f7"
git-tree-sha1 = "95bfba52e0e514336d506a6eb3488eb7d2145ad6"
uuid = "6eacf6c3-8458-43b9-ae03-caf5306d3d53"
version = "0.1.13"
version = "0.4.0"

[[deps.CSV]]
deps = ["Dates", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode"]
git-tree-sha1 = "b83aa3f513be680454437a0eee21001607e5d983"
deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings"]
git-tree-sha1 = "9519274b50500b8029973d241d32cfbf0b127d97"
uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
version = "0.8.5"
version = "0.10.2"

[[deps.ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "54fc4400de6e5c3e27be6047da2ef6ba355511f8"
git-tree-sha1 = "f9982ef575e19b0e5c7a98c6e75ee496c0f73a93"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.11.6"
version = "1.12.0"

[[deps.ChangesOfVariables]]
deps = ["ChainRulesCore", "LinearAlgebra", "Test"]
git-tree-sha1 = "bf98fa45a0a4cee295de98d4c1462be26345b9a1"
uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
version = "0.1.2"

[[deps.CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.0"

[[deps.Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "44c37b4636bc54afac5c574d2d02b625349d6582"
Expand Down Expand Up @@ -75,9 +81,9 @@ version = "1.9.0"

[[deps.DataFrames]]
deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
git-tree-sha1 = "cfdfef912b7f93e4b848e80b9befdf9e331bc05a"
git-tree-sha1 = "ae02104e835f219b8930c7664b8012c93475c340"
uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
version = "1.3.1"
version = "1.3.2"

[[deps.DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
Expand Down Expand Up @@ -118,6 +124,12 @@ version = "0.27.12"
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[deps.FilePathsBase]]
deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"]
git-tree-sha1 = "04d13bfa8ef11720c24e4d840c0033d145537df7"
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
version = "0.9.17"

[[deps.Formatting]]
deps = ["Printf"]
git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
Expand All @@ -132,7 +144,7 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
deps = ["Artifacts", "Conda", "Dates", "DocStringExtensions", "HTTP", "JSON", "LazyArtifacts", "NetcdfIO", "PkgUtility", "ProgressMeter", "PyCall", "Test"]
path = ".."
uuid = "f20cf718-bf4d-4727-bc8f-485b1f283ac6"
version = "0.2.4"
version = "0.2.5"

[[deps.HDF5_jll]]
deps = ["Artifacts", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenSSL_jll", "Pkg", "Zlib_jll"]
Expand All @@ -158,6 +170,12 @@ git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[deps.InlineStrings]]
deps = ["Parsers"]
git-tree-sha1 = "61feba885fac3a407465726d0c330b3055df897f"
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
version = "1.1.2"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Expand Down Expand Up @@ -185,9 +203,9 @@ version = "1.0.0"

[[deps.JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "22df5b96feef82434b07327e2d3c770a9b21e023"
git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.4.0"
version = "1.4.1"

[[deps.JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
Expand Down Expand Up @@ -224,9 +242,9 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[deps.Literate]]
deps = ["Base64", "IOCapture", "JSON", "REPL"]
git-tree-sha1 = "d3493acfb9e6aa0cff46b09773fc2342327b0feb"
git-tree-sha1 = "c1cdf09322a70311cae84ae787a18140d12baeba"
uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
version = "2.9.4"
version = "2.12.0"

[[deps.LogExpFunctions]]
deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
Expand Down Expand Up @@ -271,9 +289,9 @@ uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[deps.NCDatasets]]
deps = ["CFTime", "DataStructures", "Dates", "NetCDF_jll", "Printf"]
git-tree-sha1 = "abb576c93e7f101624aa49a7433f2fcbbb6d72f0"
git-tree-sha1 = "17e39eb5bbe564f48bdbefbd103bd3f49fcfcb9b"
uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
version = "0.11.8"
version = "0.11.9"

[[deps.NetCDF_jll]]
deps = ["Artifacts", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Pkg", "Zlib_jll", "nghttp2_jll"]
Expand All @@ -283,9 +301,9 @@ version = "400.702.400+0"

[[deps.NetcdfIO]]
deps = ["DataFrames", "DocStringExtensions", "NCDatasets", "Test"]
git-tree-sha1 = "24d46a3001701f303756f9d31cdcd66f25ee3332"
git-tree-sha1 = "5b6acd22f3552391896401960afa897ad91b2bc2"
uuid = "6a36f34d-89cf-423a-87c4-84549e82f670"
version = "0.1.2"
version = "0.2.1"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
Expand All @@ -307,19 +325,19 @@ version = "1.4.1"

[[deps.Parsers]]
deps = ["Dates"]
git-tree-sha1 = "bfd7d8c7fd87f04543810d9cbd3995972236ba1b"
git-tree-sha1 = "0b5cfbb704034b5b4c1869e36634438a047df065"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.1.2"
version = "2.2.1"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[deps.PkgUtility]]
deps = ["CLIMAParameters", "CSV", "DataFrames", "Dates", "DocStringExtensions", "LazyArtifacts", "NCDatasets", "Pkg", "Statistics", "StatsBase", "Test"]
git-tree-sha1 = "66601bd857474ec8bed2c5520613a8685f7346f0"
deps = ["CLIMAParameters", "CSV", "DataFrames", "Dates", "DocStringExtensions", "LazyArtifacts", "Pkg", "Statistics", "StatsBase", "Test"]
git-tree-sha1 = "3cbe2b4556b1746ad2db9d8f4d3eb218039efe78"
uuid = "0d262f2c-28e9-492c-8e19-d7a5c4f11611"
version = "0.1.13"
version = "0.2.0"

[[deps.PooledArrays]]
deps = ["DataAPI", "Future"]
Expand Down Expand Up @@ -436,6 +454,12 @@ uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.6"

[[deps.URIs]]
git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
Expand All @@ -449,9 +473,15 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[deps.VersionParsing]]
git-tree-sha1 = "e575cf85535c7c3292b4d89d89cc29e8c3098e47"
git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868"
uuid = "81def892-9a0e-5fdd-b105-ffc91e053289"
version = "1.2.1"
version = "1.3.0"

[[deps.WeakRefStrings]]
deps = ["DataAPI", "InlineStrings", "Parsers"]
git-tree-sha1 = "c69f9da3ff2f4f02e811c3323c22e5dfcb584cfa"
uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
version = "1.4.1"

[[deps.Zlib_jll]]
deps = ["Libdl"]
Expand Down
17 changes: 17 additions & 0 deletions src/GriddingMachine.jl
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,23 @@
#######################################################################################################################################################################################################
#
# Style Guide
# 1. Types and modules start with upper case letters for each word, such as LeafBiophysics
# 2. Functions are lower case words connected using _, such as leaf_biophysics
# 3. Constants are defined using all upper cases, such as LEAF_BIOPHYSICS
# 4. Variables are defined using all lower cases, such as leaf_bio_para
# 5. Temporary variables are defined to start with _, such as _leaf
# 6. Maximum length of each line is 200 letters (including space)
# 7. There should be 2 lines of empty lines between different components, such as two functions and methods
# 8. Bug fixes or new changes should be documented in the comments above the struct, function, or method, such as this Style Guide above Smaragdus.jl
# 9. Function parameter list that spans multiple lines need to be spaced with 12 spaces (3 tabs)
#
#######################################################################################################################################################################################################
module GriddingMachine


# include types
include("types.jl")

# include submodules
include("Blender.jl" )
include("Collector.jl" )
Expand Down
3 changes: 3 additions & 0 deletions src/Partitioner.jl
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
module Partitioner

import ..GriddingMachine: TropomiL2SIF





Expand Down
44 changes: 44 additions & 0 deletions src/types.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#######################################################################################################################################################################################################
#
# Changes to the struct
# General
# 2022-Jan-27: add struct to indicate TROPOMI SIF L2
#
#######################################################################################################################################################################################################
"""
"""
struct TropomiL2SIF
"Identifier"
ID::String
"Path to the database"
PATH::String
"Name pattern"
PATTERN::String
"If the data is stored per year"
PER_YEAR::Bool
"If the data is stored per month"
PER_MONTH::Bool
"Target folder"
TARGET::String
end


#######################################################################################################################################################################################################
#
# Changes to the construtor
# General
# 2022-Jan-27: add construtor to construt structs for TROPOMI Caltech and ESA
#
#######################################################################################################################################################################################################
"""
"""
TropomiL2SIF(id::String) = (
@assert id in ["CaltechNIR", "ESANIR"] "id must be CaltechNIR or ESANIR";

_dir = (id=="CaltechNIR" ? "/net/fluo/data3/projects/TROPOMI/nc_ungridded" : "/net/fluo/data2/data/TROPOMI_SIF_ESA/original/v2.1/l2b");
_pattern = (id=="CaltechNIR" ? "TROPO_SIF_yyyy-mm-dd_ungridded.nc" : "TROPOSIF_L2B_yyyy-mm-dd.nc");
_year = (id=="CaltechNIR" ? false : true);
_month = (id=="CaltechNIR" ? false : true);

return TropomiL2SIF(id, _dir, _pattern, _year, _month, "/home/wyujie/GriddingMachine/partitions/$(id)")
);
13 changes: 5 additions & 8 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ println();
end;


# test Requestor functions
# test Regrid functions
println();
@testset "GriddingMachine : Blender" begin
regrid(rand(720,360), 1); @test true;
Expand Down Expand Up @@ -184,11 +184,8 @@ end;
# test Requestor functions
println();
@testset "GriddingMachine : Requestor" begin
# only for high memory and storage cases, e.g., server
if Sys.islinux()
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5; interpolation=true); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5, 8); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5, 8; interpolation=true); @test true;
end;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5; interpolation=true); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5, 8); @test true;
request_LUT("LAI_MODIS_2X_8D_2017_V1", 30.5, 115.5, 8; interpolation=true); @test true;
end;

2 comments on commit 6aa4847

@Yujie-W
Copy link
Collaborator

@Yujie-W Yujie-W commented on 6aa4847 Feb 3, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/53831

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.2.5 -m "<description of version>" 6aa4847bd4864c0d5bbeb630e9093fd9ce52b695
git push origin v0.2.5

Please sign in to comment.