update BenguelaCC harvest

iodepo · Oct 29, 2024 · 627a74d · 627a74d
1 parent 0cfdd6b
commit 627a74d
Show file tree

Hide file tree

Showing 236 changed files with 3,480 additions and 4,478 deletions.
diff --git a/collection/scripts/benguelacc-harvest.py b/collection/scripts/benguelacc-harvest.py
@@ -10,7 +10,7 @@
          exposed through the OGC:CSW service, and also a
          .RDF resource file.
          
-Requires: Python 3.x
+Requires: - Python 3.x
 
 Notes:
 
@@ -24,12 +24,12 @@
 # define common variables
 CSW_ENDPOINT = "https://geodata.benguelacc.org/catalogue/csw"
 CSW_ENDPOINT_TIMEOUT = 60 #seconds
-PATH_TO_DATA_FOLDER = "./data-benguelacc/"
+PATH_TO_DATA_FOLDER = "../tempHosting/data-benguelacc/"
 NEW_RDF_FILENAME = "benguelacc-catalogue.rdf"
 HOSTNAME = "https://geodata.benguelacc.org"
 LOGFILE = "benguelacc-harvest.log"
-SHORTNAME = "benguelacc" #must be hyphen
-ID_URL_BASE = "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/"
+SHORTNAME = "benguelacc" #must not contain spaces, but can contain hyphen
+ID_URL_BASE = "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/"
 
 """
 #########################
@@ -42,8 +42,6 @@
 import os, sys, io, uuid
 from owslib.csw import CatalogueServiceWeb
 from owslib.fes import SortBy, SortProperty
-import ssl
-import pandas as pd
 import kglab
 import logging
 
@@ -52,13 +50,6 @@
                     format="%(asctime)s;%(levelname)s;%(message)s",  
                     datefmt="%Y-%m-%d %H:%M", filemode = "w")
 
-# generate a Context for each connection
-# disable SSL for now
-
-ctx = ssl.create_default_context()
-ctx.check_hostname = False
-ctx.verify_mode = ssl.CERT_NONE
-
 # prepare namespace
 
 namespaces = {
@@ -159,7 +150,17 @@
             maxx = csw.records[rec].identification.bbox.maxx
             maxy = csw.records[rec].identification.bbox.maxy
 
-            poly = str("""POLYGON(({} {}, {} {}, {} {}, {} {}, {} {}))""".format(minx, miny, minx, maxy, maxx, maxy, maxx, miny, minx, miny))
+            #poly = str("""POLYGON(({} {}, {} {}, {} {}, {} {}, {} {}))""".format(minx, miny, minx, maxy, maxx, maxy, maxx, miny, minx, miny))
+
+            #schema.org expects lat long (Y X) coordinate order
+            boxCoords = str("""{} {} {} {}""".format(miny, minx, maxy, maxx))
+            print("    GeoShape:Box: " + boxCoords)
+            spatialCov = {}
+            spatialCov["@type"] = "https://schema.org/Place"
+            geo = {}
+            geo["@type"] = "https://schema.org/GeoShape"
+            geo["https://schema.org/box"] = boxCoords 
+            spatialCov["https://schema.org/geo"] = geo
 
             data = {}
 
@@ -177,19 +178,7 @@
             data["https://schema.org/description"] = description
             data["https://schema.org/url"] = url
 
-            aswkt = {}
-            aswkt["@type"] = "http://www.opengis.net/ont/geosparql#wktLiteral"
-            aswkt["@value"] = poly
-
-            crs = {}
-            crs["@id"] = "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-
-            hg = {}
-            hg["@type"] = "http://www.opengis.net/ont/sf#Polygon" 
-            hg["http://www.opengis.net/ont/geosparql#asWKT"] = aswkt
-            hg["http://www.opengis.net/ont/geosparql#crs"] = crs
-
-            data["http://www.opengis.net/ont/geosparql#hasGeometry"] = hg
+            data["https://schema.org/spatialCoverage"] = spatialCov 
 
             # keyword(s) loop
             k = ""
@@ -212,7 +201,7 @@
             k_list = k.split(",")
             data["https://schema.org/keywords"] = k_list
 
-            context = {"@vocab": "https://schema.org/", "geosparql": "http://www.opengis.net/ont/geosparql#"}
+            context = {"@vocab": "https://schema.org/"}
             compacted = jsonld.compact(data, context)
 
             # need sha hash for the "compacted" var and then also generate the prov for this record.

diff --git a/collection/tempHosting/data-benguelacc/benguelacc-000a6aa2-5f46-11ec-9cf2-0242ac1c0006.json b/collection/tempHosting/data-benguelacc/benguelacc-000a6aa2-5f46-11ec-9cf2-0242ac1c0006.json
@@ -1,25 +1,21 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-000a6aa2-5f46-11ec-9cf2-0242ac1c0006.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-000a6aa2-5f46-11ec-9cf2-0242ac1c0006.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((10.0565643310547 -29.9711952209473, 10.0565643310547 -18.7997817993164, 14.9446954727173 -18.7997817993164, 14.9446954727173 -29.9711952209473, 10.0565643310547 -29.9711952209473))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "Terraces  found in Namibian sea.\nDATA SOURCE: Harris, P.T., Macmillan-Lawler, M., Rupp, J., Baker, E.K. 2014. Geomorphology of the oceans. Marine Geology, 352: 4-24. GEBCO (General Bathymetric Chart of the Oceans) Available at: http://www.gebco.net/data_and_poducts/gridded_bathymetry_data/. \nGeomorphology map from www.bluehabitats.org\nMETHODS: Slight manual revision of the Blue Habitats / Harris 2014 global geomorphology\nmap.",
     "keywords": [
         "Current Status Report",
         "Namibia"
     ],
     "name": "NAM CSR Terraces",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-29.9711952209473 10.0565643310547 -18.7997817993164 14.9446954727173"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:NAM_CSR_GREE_terraces"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-01062a52-8f09-11ec-8819-0242ac1c0006.json b/collection/tempHosting/data-benguelacc/benguelacc-01062a52-8f09-11ec-8819-0242ac1c0006.json
@@ -1,25 +1,21 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-01062a52-8f09-11ec-8819-0242ac1c0006.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-01062a52-8f09-11ec-8819-0242ac1c0006.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((8.231640522 -30.658138482, 8.231640522 -17.246196504, 17.073816534 -17.246196504, 17.073816534 -30.658138482, 8.231640522 -30.658138482))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "DESCRIPTION: Shipping is an important use of marine areas in Namibia. Associated\nimpacts from shipping are underwater noise pollution, oil spillage and other pollutants,\npropeller and wake damage in shallow areas, dumping of waste (especially plastic) into\nthe ocean, direct strikes on cetaceans, invasion pathways for alien species and associated\ninfrastructure (ports and anchorages).",
     "keywords": [
         "Current Status Report",
         "Namibia"
     ],
     "name": "NAM CSR Shipping intensity",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-30.658138482 8.231640522 -17.246196504 17.073816534"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/NAM_CSR_MTP_shipping_intensity:geonode:NAM_CSR_MTP_shipping_intensity"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-01e050a6-67c7-11ec-9de4-0242ac1c0006.json b/collection/tempHosting/data-benguelacc/benguelacc-01e050a6-67c7-11ec-9de4-0242ac1c0006.json
@@ -1,26 +1,22 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-01e050a6-67c7-11ec-9de4-0242ac1c0006.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-01e050a6-67c7-11ec-9de4-0242ac1c0006.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((14.4294900894165 -22.9716682434082, 14.4294900894165 -22.7771892547607, 14.5439987182617 -22.7771892547607, 14.5439987182617 -22.9716682434082, 14.4294900894165 -22.9716682434082))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "Areas delineated for  inshore traffic in the Walvis Bay port channel.",
     "keywords": [
         "Current Status Report",
         "Port",
         "Namibia"
     ],
     "name": "NAM CSR Inshore traffic zone",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-22.9716682434082 14.4294900894165 -22.7771892547607 14.5439987182617"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:NAM_CSR_MTP_inshore_traffic_zone"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-0220de3a-68a5-11ec-803f-0242ac1c0006.json b/collection/tempHosting/data-benguelacc/benguelacc-0220de3a-68a5-11ec-803f-0242ac1c0006.json
@@ -1,26 +1,22 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-0220de3a-68a5-11ec-803f-0242ac1c0006.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-0220de3a-68a5-11ec-803f-0242ac1c0006.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((11.6833324432373 -23.0, 11.6833324432373 -18.0, 14.0500001907349 -18.0, 14.0500001907349 -23.0, 11.6833324432373 -23.0))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "Oceanographic moorings deployed by the Baltic Sea Research Institute in collaboration with MFMR. Moorings collect data on temperature, salinity, oxygen and ocean currents.",
     "keywords": [
         "Current Status Report",
         "Research",
         "Namibia"
     ],
     "name": "NAM CSR Moorings",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-23.0 11.6833324432373 -18.0 14.0500001907349"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:NAM_CSR_RES_Moorings"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-03fac06c-13eb-11ed-b034-0242ac1c0005.json b/collection/tempHosting/data-benguelacc/benguelacc-03fac06c-13eb-11ed-b034-0242ac1c0005.json
@@ -1,26 +1,21 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-03fac06c-13eb-11ed-b034-0242ac1c0005.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-03fac06c-13eb-11ed-b034-0242ac1c0005.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((13,0036296844482 -9,33810043334961, 13,0036296844482 -9,3244743347168, 13,1185398101807 -9,3244743347168, 13,1185398101807 -9,33810043334961, 13,0036296844482 -9,33810043334961))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
-    "description": "No abstract provided",
+    "description": "Cabos e Ductos Submarinos. Método –Transformou-se as coordenadas para graus decimais e a posterior tratou-se os mesmos no Qgis.",
     "keywords": [
-        "AGO_OEM_INFR_WACS",
-        "features",
-        "Global"
+        "Cabos",
+        "Angola"
     ],
-    "name": "AGO_OEM_INFR_WACS",
+    "name": "AGO Cabos Submarinos WACS 2021",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-9.33810043334961 13.0036296844482 -9.3244743347168 13.1185398101807"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:AGO_OEM_INFR_WACS"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-0549fd70-0469-11ed-b6be-0242ac1c0005.json b/collection/tempHosting/data-benguelacc/benguelacc-0549fd70-0469-11ed-b6be-0242ac1c0005.json
@@ -1,26 +1,21 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-0549fd70-0469-11ed-b6be-0242ac1c0005.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-0549fd70-0469-11ed-b6be-0242ac1c0005.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((11,766676902771 -17,4416732788086, 11,766676902771 -4,83762121200562, 17,6371536254883 -4,83762121200562, 17,6371536254883 -17,4416732788086, 11,766676902771 -17,4416732788086))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
-    "description": "No abstract provided",
+    "description": "Rios. Método-Shapefile Baixada.",
     "keywords": [
-        "AGO_OEM_BASE_RIOS",
-        "features",
-        "Global"
+        "Rios",
+        "Angola"
     ],
-    "name": "AGO_OEM_BASE_RIOS",
+    "name": "AGO Rios 2021",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-17.4416732788086 11.766676902771 -4.83762121200562 17.6371536254883"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:AGO_OEM_BASE_RIOS"
 }
diff --git a/collection/tempHosting/data-benguelacc/benguelacc-05bc3598-6333-11ec-8eab-0242ac1c0006.json b/collection/tempHosting/data-benguelacc/benguelacc-05bc3598-6333-11ec-8eab-0242ac1c0006.json
@@ -1,26 +1,22 @@
 {
     "@context": {
-        "@vocab": "https://schema.org/",
-        "geosparql": "http://www.opengis.net/ont/geosparql#"
+        "@vocab": "https://schema.org/"
     },
-    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/schema-dev-jm/code/notebooks/Exploration/data-benguelacc/benguelacc-05bc3598-6333-11ec-8eab-0242ac1c0006.json",
+    "@id": "https://raw.githubusercontent.com/iodepo/odis-arch/master/collection/tempHosting/data-benguelacc/benguelacc-05bc3598-6333-11ec-8eab-0242ac1c0006.json",
     "@type": "Dataset",
-    "geosparql:hasGeometry": {
-        "@type": "http://www.opengis.net/ont/sf#Polygon",
-        "geosparql:asWKT": {
-            "@type": "geosparql:wktLiteral",
-            "@value": "POLYGON((14.5083322525024 -22.9295749664307, 14.5083322525024 -22.9133644104004, 14.5284357070923 -22.9133644104004, 14.5284357070923 -22.9295749664307, 14.5083322525024 -22.9295749664307))"
-        },
-        "geosparql:crs": {
-            "@id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
-        }
-    },
     "description": "Areas of interest or reserved for naval defence, including military training areas, ammunition dumping area and naval infrastructure area",
     "keywords": [
         "Current Status Report",
         "Military",
         "Namibia"
     ],
     "name": "NAM CSR Military restricted area III",
+    "spatialCoverage": {
+        "@type": "Place",
+        "geo": {
+            "@type": "GeoShape",
+            "box": "-22.9295749664307 14.5083322525024 -22.9133644104004 14.5284357070923"
+        }
+    },
     "url": "https://geodata.benguelacc.org/layers/geonode_deploy_data:geonode:NAM_CSR_DEF_naval_infrastructure_area4"
 }