diff --git a/data/yaml/mspass.yaml b/data/yaml/mspass.yaml index 7ecd265fb..653a59e1a 100644 --- a/data/yaml/mspass.yaml +++ b/data/yaml/mspass.yaml @@ -66,6 +66,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. diff --git a/data/yaml/mspass_fdsn.yaml b/data/yaml/mspass_fdsn.yaml index fa35813e5..cc8ef2e9a 100644 --- a/data/yaml/mspass_fdsn.yaml +++ b/data/yaml/mspass_fdsn.yaml @@ -62,6 +62,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. diff --git a/data/yaml/mspass_lite.yaml b/data/yaml/mspass_lite.yaml index 66d2e3c1b..f70055f2a 100644 --- a/data/yaml/mspass_lite.yaml +++ b/data/yaml/mspass_lite.yaml @@ -37,6 +37,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. @@ -283,4 +287,5 @@ Metadata: readonly: false loc: collection: wf_Seismogram - readonly: false \ No newline at end of file + readonly: false + diff --git a/data/yaml/mspass_s3.yaml b/data/yaml/mspass_s3.yaml index eb11730fc..2473d8607 100644 --- a/data/yaml/mspass_s3.yaml +++ b/data/yaml/mspass_s3.yaml @@ -62,6 +62,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index d17f265e4..92d2bae8c 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -1,4 +1,3 @@ - import os import io import copy @@ -4960,13 +4959,13 @@ def save_inventory(self, inv, networks_to_exclude=["SY"], verbose=False): loc_etime = self._handle_null_endtime(loc_etime) rec["lat"] = loc_lat rec["lon"] = loc_lon - # save coordinates in both geoJSON and "legacy" - # format - rec["coords"] = [loc_lon,loc_lat] - # Illegal lon,lat values will cause this to throw a - # ValueError exception. We let it do that as + # save coordinates in both geoJSON and "legacy" + # format + rec["coords"] = [loc_lon, loc_lat] + # Illegal lon,lat values will cause this to throw a + # ValueError exception. We let it do that as # it indicates a problem datum - rec = geoJSON_doc(loc_lat,loc_lon,doc=rec,key='location') + rec = geoJSON_doc(loc_lat, loc_lon, doc=rec, key="location") rec["elev"] = loc_elev rec["edepth"] = loc_edepth rec["starttime"] = starttime.timestamp @@ -5493,12 +5492,12 @@ def save_catalog(self, cat, verbose=False): rec["lat"] = o.latitude rec["lon"] = o.longitude # save the epicenter data in both legacy format an d - # geoJSON format. Either can technically be used in a - # geospatial query but the geoJSON version is always + # geoJSON format. Either can technically be used in a + # geospatial query but the geoJSON version is always # preferred rec["coords"] = [o.longitude, o.latitude] # note this function updates rec and returns the upodate - rec = geoJSON_doc(o.latitude, o.longitude, doc=rec,key='epicenter') + rec = geoJSON_doc(o.latitude, o.longitude, doc=rec, key="epicenter") # It appears quakeml puts source depths in meter # convert to km # also obspy's catalog object seesm to allow depth to be @@ -6505,6 +6504,12 @@ def _atomic_save_all_documents( # Warning: this can lead to sample data save orphans mspass_object.kill() + # Always set starttime and endtime + if not exclude_keys or "starttime" not in exclude_keys: + insertion_dict["starttime"] = mspass_object.t0 + if not exclude_keys or "endtime" not in exclude_keys: + insertion_dict["endtime"] = mspass_object.endtime() + # add tag - intentionally not set in mspass_object returned if data_tag: insertion_dict["data_tag"] = data_tag @@ -8186,54 +8191,57 @@ def _erase_normalized( mdout.erase(k) return mdout -def geoJSON_doc(lat,lon,doc=None,key='epicenter')->dict: + +def geoJSON_doc(lat, lon, doc=None, key="epicenter") -> dict: """ Convenience function to create a geoJSON format point object document - from a points specified by latitude and longitude. The format - for a geoJSON point isn't that strange but how to structure it into + from a points specified by latitude and longitude. The format + for a geoJSON point isn't that strange but how to structure it into a mongoDB document for use with geospatial queries is not as - clear from current MongoDB documentation. This function makes that + clear from current MongoDB documentation. This function makes that proess easier. - - The required inpput is latitude (lat) and longitude (lon). The + + The required inpput is latitude (lat) and longitude (lon). The values are assumed to be in degrees for compatibility with MongoDB. - That means latitude must be -90<=lat<=90 and longitude - must satisfy -180<=lat<=180. The function will try to handle the - common situation with 0<=lon<=360 by wrapping 90->180 values to - -180->0, A ValueError exception is thrown if + That means latitude must be -90<=lat<=90 and longitude + must satisfy -180<=lat<=180. The function will try to handle the + common situation with 0<=lon<=360 by wrapping 90->180 values to + -180->0, A ValueError exception is thrown if for any other situation with lot or lon outside those bounds. - - If you specify the optional "doc" argument it is assumed to be - a python dict to which the geoJSON point data is to be added. - By default a new dict is created that will contain only the - geoJSON point data. The doc options is useful if you want to - add the geoJSON data to the document before appending it. - The default is more useful for updates to add geospatial - query capabilities to a collection with lat-lon data that is + + If you specify the optional "doc" argument it is assumed to be + a python dict to which the geoJSON point data is to be added. + By default a new dict is created that will contain only the + geoJSON point data. The doc options is useful if you want to + add the geoJSON data to the document before appending it. + The default is more useful for updates to add geospatial + query capabilities to a collection with lat-lon data that is not properl structure. In all cases the geoJSON data is a - itself a python dict but a value associated accessible from + itself a python dict but a value associated accessible from the output dict with te key defined by the "key" argument - (default is 'epicenter', which is appropriate for earthquake + (default is 'epicenter', which is appropriate for earthquake source data.) - with a + with a """ outval = dict() - outval['type'] = 'Point' + outval["type"] = "Point" lon = float(lon) # make this it is a float for database consistency - if lon>180.0 and lon<=360.0: + if lon > 180.0 and lon <= 360.0: # we do this correction silently lon -= 360.0 lat = float(lat) if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0: message = "geoJSON_doc: Illegal geographic input\n" message += "latitude received={} MongoDB requires [-90,90] range\n".format(lat) - message += "longitude received={} MongoDB requires [-180,180] range".format(lon) + message += "longitude received={} MongoDB requires [-180,180] range".format( + lon + ) raise ValueError(message) - - outval['coordinates'] = [float(lon), float(lat)] + + outval["coordinates"] = [float(lon), float(lat)] if doc: retdoc = doc retdoc[key] = outval else: - retdoc={key : outval} + retdoc = {key: outval} return retdoc diff --git a/python/tests/db/test_database.py b/python/tests/db/test_database.py index 9dd73fdb9..a6d526448 100644 --- a/python/tests/db/test_database.py +++ b/python/tests/db/test_database.py @@ -49,7 +49,7 @@ from mspasspy.util import logging_helper from bson.objectid import ObjectId from datetime import datetime -from mspasspy.db.database import Database,geoJSON_doc +from mspasspy.db.database import Database, geoJSON_doc from mspasspy.db.client import DBClient @@ -2128,13 +2128,13 @@ def test_check_undefined_keys(self): ts, mode="promiscuous", storage_mode="gridfs", - exclude_keys=["extra2", "starttime"], + exclude_keys=["extra2", "calib"], return_data=True, ) assert save_res.live self.db["wf_TimeSeries"].update_one({"_id": ts["_id"]}, {"$set": {"t0": 1.0}}) res = self.db["wf_TimeSeries"].find_one({"_id": ts["_id"]}) - assert "starttime" not in res + assert "calib" not in res assert "npts" not in res assert "t0" in res @@ -3451,51 +3451,49 @@ def test_set_schema(self): self.db.set_schema("mspass_lite.yaml") with pytest.raises(KeyError, match="site"): self.db.database_schema._attr_dict["site"] - + def test_geoJSON_doc(self): """ - Tests only the function geoJSON_doc added Jan 2024 to properly - create geoJSON records that allow geospatial queries in - site, channel, and source. That function is now used in - Database.save_inventory and Database.save_catalog to always - save a geoJSOn format location data. We assume the bulk of the - code is already tested when those two functions are tested in - other test functions. The big thing here is testing the + Tests only the function geoJSON_doc added Jan 2024 to properly + create geoJSON records that allow geospatial queries in + site, channel, and source. That function is now used in + Database.save_inventory and Database.save_catalog to always + save a geoJSOn format location data. We assume the bulk of the + code is already tested when those two functions are tested in + other test functions. The big thing here is testing the bad data input handlers. """ # test null doc input - doc = geoJSON_doc(22.0,44.0,key='testpoint') - assert 'testpoint' in doc - val = doc['testpoint'] - assert val['type'] == "Point" - coords = val['coordinates'] + doc = geoJSON_doc(22.0, 44.0, key="testpoint") + assert "testpoint" in doc + val = doc["testpoint"] + assert val["type"] == "Point" + coords = val["coordinates"] # note coordinates pair is lon,lat assert coords[0] == 44.0 assert coords[1] == 22.0 - doc = geoJSON_doc(33.0,55.0,doc=doc,key='testpoint2') + doc = geoJSON_doc(33.0, 55.0, doc=doc, key="testpoint2") # doc should contain both testpoint and testpoint2 # This tests update feature of this function - assert 'testpoint' in doc - assert 'testpoint2' in doc - val = doc['testpoint2'] - assert val['type'] == "Point" - coords = val['coordinates'] + assert "testpoint" in doc + assert "testpoint2" in doc + val = doc["testpoint2"] + assert val["type"] == "Point" + coords = val["coordinates"] # note coordinates pair is lon,lat assert coords[0] == 55.0 assert coords[1] == 33.0 # recoverable value test - doc = geoJSON_doc(20,270.0,key='recoverable') - assert 'recoverable' in doc - val = doc['recoverable'] - coords = val['coordinates'] - # use is_close because -90 is computed but normal floating point math + doc = geoJSON_doc(20, 270.0, key="recoverable") + assert "recoverable" in doc + val = doc["recoverable"] + coords = val["coordinates"] + # use is_close because -90 is computed but normal floating point math # for that simple calculation would allow an == to also work - assert np.isclose(coords[0],-90.0) + assert np.isclose(coords[0], -90.0) assert coords[1] == 20.0 - - with pytest.raises(ValueError, - match="Illegal geographic input"): - doc = geoJSON_doc(20,400) + with pytest.raises(ValueError, match="Illegal geographic input"): + doc = geoJSON_doc(20, 400)