Switch from Artifacts to Scratchspaces (#13)

* Switch from Artifacts to Scratchspaces

since we would need a builder repo to create tarballs.

* Fully fix up

* Test from Julia 1.3 up

* ai I guess

* fix bathymetry

* Test only on 1.6+

GeoJSON version is too old on 1.3.

.github/workflows/CI.yml

@@ -18,8 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
-          - '1.3'
+          - '1.6'
           - '1.8'
           - 'nightly'
         os:

Project.toml

@@ -4,11 +4,16 @@ authors = ["Haakon Ludvig Langeland Ervik (@haakon-e), Anshul Singhvi <anshulsin
 version = "0.1.0"
 
 [deps]
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GeoJSON = "61d90e0f-e114-555e-ac52-39dfb47a3ef9"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 
 [compat]
+Downloads = "1"
 GeoJSON = "0.6, 0.7, 0.8"
+Pkg = "1"
+Scratch = "1"
 julia = "1.3"
 
 [extras]

README.md

@@ -9,7 +9,7 @@ This package provides a Julia interface to the [Natural Earth](http://www.natura
 
 Currently, this package provides a single function, `naturalearth`, which fetches any `.geojson` file from [this](https://github.com/nvkelso/natural-earth-vector/tree/master/geojson) repository. The function returns a `GeoJSON.FeatureCollection` object.
 
-The data is downloaded on demand and cached using Julia's `Artifacts` system. This means that the first time you fetch a dataset, it will take a while to download, and you will need an internet connection. Subsequent calls will be much faster, even in a new session.
+The data is downloaded on demand and cached using Julia's scratch space system. This means that the first time you fetch a dataset, it will take a while to download, and you will need an internet connection. Subsequent calls will be much faster, even in a new session.
 
 ## Acknowledgements
 

src/NaturalEarth.jl

@@ -1,54 +1,102 @@
 module NaturalEarth
 
-import GeoJSON
-using Pkg
-using Pkg.Artifacts
+import GeoJSON, Downloads
+using Scratch
 
-const available_artifacts = collect(keys(
-    Artifacts.select_downloadable_artifacts(Artifacts.find_artifacts_toml(@__FILE__); include_lazy=true)
-))
+const naturalearth_cache = Ref{String}("")
+
+function __init__()
+    # Populate the cache by obtaining a directory for the 
+    # scratchspace.
+    global naturalearth_cache
+    naturalearth_cache[] = @get_scratch!("naturalearth")
+end
 
 export naturalearth, bathymetry
 
 """
-    naturalearth(name::String)
+    naturalearth(name::String; version = v"5.1.2")
+    naturalearth(name::String, scale::Int; version = v"5.1.2")
 
 Load a NaturalEarth dataset as a `GeoJSON.FeatureCollection` object.
 
-Valid names are found in `Artifacts.toml`. 
-We aim to support all datasets listed in https://github.com/nvkelso/natural-earth-vector/tree/master/geojson
+The `name` should not include the `ne_` prefix, and if providing a `scale` should also not include a scale.  No suffix should be added.
+
+For example, to get the `ne_110m_admin_0_countries` dataset, you could use:
+
+```julia
+naturalearth("admin_0_countries", 110)
+naturalearth("110m_admin_0_countries")
+```
+
+We aim to support all datasets listed in https://github.com/nvkelso/natural-earth-vector/tree/master/geojson.
+
+!!! warning
+    This function downloads files from the Internet when not found locally.
 """
-function naturalearth(name::String)
-    pth = @artifact_str("$name/$name.geojson")
-    @assert isfile(pth) "`$name` is not a valid NaturalEarth.jl artifact"
-    GeoJSON.read(read(pth, String))
+function naturalearth(full_name::String; version::Union{VersionNumber, String} = v"5.1.2")
+    filename = full_name
+    # Ensure `ne` prefix
+    if !startswith(full_name, "ne_")
+        filename = "ne_" * filename
+    end
+    # and `.geojson` suffix
+    if !endswith(filename, ".geojson")
+        filename = filename * ".geojson"
+    end
+    # Create a String object from the version number
+    version_string = version isa VersionNumber ? "v" * string(version) : version
+
+    # First, check that the appropriate path exists
+    if !ispath(joinpath(naturalearth_cache[], version_string))
+        mkpath(joinpath(naturalearth_cache[], version_string))
+    end
+    # Check that version's cache before downloading
+    filepath = joinpath(naturalearth_cache[], version_string, filename)
+    if !isfile(filepath)
+        # Download from Githack CDN
+        # We could change this later, to use zipped Shapefiles and return a GeoDataFrame or something
+        try
+            Downloads.download("https://rawcdn.githack.com/nvkelso/natural-earth-vector/$version_string/geojson/$filename", filepath)
+        catch e
+            if e isa Downloads.RequestError
+                @error("NaturalEarth.jl: Could not download file $filename. Check the name and try again.")
+                rethrow(e)
+            else
+                rethrow(e)
+            end
+        end
+    end
+    return GeoJSON.read(read(filepath, String))
 end
 
+function naturalearth(name::String, scale::Int)
+    @assert scale in (10, 50, 110) "`scale` must be one of 10, 50, or 110.  Got $scale."
+
+    return naturalearth("$(scale)m_$(name)")
+end
 
 """
     bathymetry(contour::Int = 2000)
 
 Convenient acccess to ocean bathymetry datasets.
 Currently tested on: https://www.naturalearthdata.com/downloads/10m-physical-vectors/10m-bathymetry/
+
 The function returns a MultiPolygon describing the bathymetry at a given depth contour.
-The following depths should be available: [10000, 9000, 8000, 7000, 6000, 5000, 4000, 3000, 2000, 1000, 200, 0]
+The following depths should be available: `[10000, 9000, 8000, 7000, 6000, 5000, 4000, 3000, 2000, 1000, 200, 0]`
 """
 function bathymetry(contour::Int=2000)
-    global available_artifacts
     # extract a list of all available bathymetry files
-    bathyfiles = filter(contains("bathymetry"), available_artifacts)
-    # Extract depth signifier from filename
-    matches = match.(r"ne_10m_.*_(\d+)", bathyfiles)
-    depths = parse.(Int, first.(getproperty.(matches, :captures)))
+    available_depths = [10000, 9000, 8000, 7000, 6000, 5000, 4000, 3000, 2000, 1000, 200, 0]
     # Extract the file corresponding to the contour
-    fileind = findfirst(==(contour), depths)
-    isnothing(fileind) && error("Contour $contour not found. Available contours: $(sort(depths))")
-
-    # Open bathymetry file
-    bathyfile = bathyfiles[fileind]
-    return naturalearth(bathyfile)
+    fileind = findfirst(==(contour), available_depths)
+    isnothing(fileind) && error("Contour $contour not found. Available contours: \n$(available_depths)")
+    # Open bathymetry file.  They are prefixed by a letter corresponding to depth in reverse order from A to K,
+    # so we perform a bit of arithmetic to obtain that.
+    return naturalearth("10m_bathymetry_$('A' + (fileind - 1))_$(contour)")
 end
 
+geojson_file_name(name, scale) = "ne_$(scale)m_$(name).geojson"
 
 
 end  # end module

test/runtests.jl

@@ -5,6 +5,9 @@ using Test
     # Write your tests here.
     @testset "Bathymetry" begin
         @test_throws "Available contours" bathymetry(10000000)
-        @test_nowarn bathymetry(4000)
+        # This also tests the whole pipeline...
+        @test bathymetry(4000) isa NaturalEarth.GeoJSON.FeatureCollection
     end
+    # This tests for an error in filename.
+    @test_throws "404" naturalearth("asfhcsakdlfjnskfas")
 end