From ecad47bbc7f50a0fb319a06f0b8f8d1e15f352fb Mon Sep 17 00:00:00 2001 From: Gary Pavlis Date: Wed, 17 Jan 2024 09:21:08 -0500 Subject: [PATCH 1/7] Fix bug in handling starttime and endtime Previous version was failing to set starttime and endtime on saves. Also data loaded and cut had incorrect starttime and endtimes saved because they werne't being replaced. --- python/mspasspy/db/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index d17f265e4..5d600994d 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -6508,6 +6508,9 @@ def _atomic_save_all_documents( # add tag - intentionally not set in mspass_object returned if data_tag: insertion_dict["data_tag"] = data_tag + # Always set starttime and endtime + insertion_dict["starttime"] = mspass_object.t0 + insertion_dict("endtime"] = mspass_object.endtime() else: # We need to clear data tag if was previously defined in # this case or a the old tag will be saved with this datum From cc4fb2e8facc344e2ff88509612f8fa841c07b6e Mon Sep 17 00:00:00 2001 From: wangyinz Date: Wed, 17 Jan 2024 09:43:42 -0600 Subject: [PATCH 2/7] fix typo --- python/mspasspy/db/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index 5d600994d..b4a97bdb0 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -6510,7 +6510,7 @@ def _atomic_save_all_documents( insertion_dict["data_tag"] = data_tag # Always set starttime and endtime insertion_dict["starttime"] = mspass_object.t0 - insertion_dict("endtime"] = mspass_object.endtime() + insertion_dict["endtime"] = mspass_object.endtime() else: # We need to clear data tag if was previously defined in # this case or a the old tag will be saved with this datum From 38f9210fda868c2d2bebcc970427ed9661a43579 Mon Sep 17 00:00:00 2001 From: wangyinz Date: Wed, 17 Jan 2024 09:25:53 -0600 Subject: [PATCH 3/7] add endtime to schema --- data/yaml/mspass.yaml | 4 ++++ data/yaml/mspass_fdsn.yaml | 4 ++++ data/yaml/mspass_lite.yaml | 7 ++++++- data/yaml/mspass_s3.yaml | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/data/yaml/mspass.yaml b/data/yaml/mspass.yaml index 7ecd265fb..653a59e1a 100644 --- a/data/yaml/mspass.yaml +++ b/data/yaml/mspass.yaml @@ -66,6 +66,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. diff --git a/data/yaml/mspass_fdsn.yaml b/data/yaml/mspass_fdsn.yaml index fa35813e5..cc8ef2e9a 100644 --- a/data/yaml/mspass_fdsn.yaml +++ b/data/yaml/mspass_fdsn.yaml @@ -62,6 +62,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. diff --git a/data/yaml/mspass_lite.yaml b/data/yaml/mspass_lite.yaml index 66d2e3c1b..f70055f2a 100644 --- a/data/yaml/mspass_lite.yaml +++ b/data/yaml/mspass_lite.yaml @@ -37,6 +37,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. @@ -283,4 +287,5 @@ Metadata: readonly: false loc: collection: wf_Seismogram - readonly: false \ No newline at end of file + readonly: false + diff --git a/data/yaml/mspass_s3.yaml b/data/yaml/mspass_s3.yaml index eb11730fc..2473d8607 100644 --- a/data/yaml/mspass_s3.yaml +++ b/data/yaml/mspass_s3.yaml @@ -62,6 +62,10 @@ Database: concept: Time shift applied to define relative time of 0.0 aliases: t0shift constraint: required + endtime: + type: double + concept: Time of last sample of data (epoch time or relative to some other time mark) + constraint: optional utc_convertible: type: bool concept: When true starttime_shift can be used to convert relative time to UTC. From 55494356f0936ec8e1b5335d532704d18cb85651 Mon Sep 17 00:00:00 2001 From: wangyinz Date: Wed, 17 Jan 2024 09:53:32 -0600 Subject: [PATCH 4/7] fix placing new code in the wrong block --- python/mspasspy/db/database.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index b4a97bdb0..a7e105717 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -6505,12 +6505,13 @@ def _atomic_save_all_documents( # Warning: this can lead to sample data save orphans mspass_object.kill() - # add tag - intentionally not set in mspass_object returned - if data_tag: - insertion_dict["data_tag"] = data_tag # Always set starttime and endtime insertion_dict["starttime"] = mspass_object.t0 insertion_dict["endtime"] = mspass_object.endtime() + + # add tag - intentionally not set in mspass_object returned + if data_tag: + insertion_dict["data_tag"] = data_tag else: # We need to clear data tag if was previously defined in # this case or a the old tag will be saved with this datum From 65d0d745aa493ba5bc3f34683c6f17bb260b21b9 Mon Sep 17 00:00:00 2001 From: wangyinz Date: Wed, 17 Jan 2024 15:54:14 +0000 Subject: [PATCH 5/7] :art: Format Python code with psf/black --- python/mspasspy/db/database.py | 78 ++++++++++++++++---------------- python/tests/db/test_database.py | 58 ++++++++++++------------ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index a7e105717..70d764880 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -1,4 +1,3 @@ - import os import io import copy @@ -4960,13 +4959,13 @@ def save_inventory(self, inv, networks_to_exclude=["SY"], verbose=False): loc_etime = self._handle_null_endtime(loc_etime) rec["lat"] = loc_lat rec["lon"] = loc_lon - # save coordinates in both geoJSON and "legacy" - # format - rec["coords"] = [loc_lon,loc_lat] - # Illegal lon,lat values will cause this to throw a - # ValueError exception. We let it do that as + # save coordinates in both geoJSON and "legacy" + # format + rec["coords"] = [loc_lon, loc_lat] + # Illegal lon,lat values will cause this to throw a + # ValueError exception. We let it do that as # it indicates a problem datum - rec = geoJSON_doc(loc_lat,loc_lon,doc=rec,key='location') + rec = geoJSON_doc(loc_lat, loc_lon, doc=rec, key="location") rec["elev"] = loc_elev rec["edepth"] = loc_edepth rec["starttime"] = starttime.timestamp @@ -5493,12 +5492,12 @@ def save_catalog(self, cat, verbose=False): rec["lat"] = o.latitude rec["lon"] = o.longitude # save the epicenter data in both legacy format an d - # geoJSON format. Either can technically be used in a - # geospatial query but the geoJSON version is always + # geoJSON format. Either can technically be used in a + # geospatial query but the geoJSON version is always # preferred rec["coords"] = [o.longitude, o.latitude] # note this function updates rec and returns the upodate - rec = geoJSON_doc(o.latitude, o.longitude, doc=rec,key='epicenter') + rec = geoJSON_doc(o.latitude, o.longitude, doc=rec, key="epicenter") # It appears quakeml puts source depths in meter # convert to km # also obspy's catalog object seesm to allow depth to be @@ -6505,7 +6504,7 @@ def _atomic_save_all_documents( # Warning: this can lead to sample data save orphans mspass_object.kill() - # Always set starttime and endtime + # Always set starttime and endtime insertion_dict["starttime"] = mspass_object.t0 insertion_dict["endtime"] = mspass_object.endtime() @@ -8190,54 +8189,57 @@ def _erase_normalized( mdout.erase(k) return mdout -def geoJSON_doc(lat,lon,doc=None,key='epicenter')->dict: + +def geoJSON_doc(lat, lon, doc=None, key="epicenter") -> dict: """ Convenience function to create a geoJSON format point object document - from a points specified by latitude and longitude. The format - for a geoJSON point isn't that strange but how to structure it into + from a points specified by latitude and longitude. The format + for a geoJSON point isn't that strange but how to structure it into a mongoDB document for use with geospatial queries is not as - clear from current MongoDB documentation. This function makes that + clear from current MongoDB documentation. This function makes that proess easier. - - The required inpput is latitude (lat) and longitude (lon). The + + The required inpput is latitude (lat) and longitude (lon). The values are assumed to be in degrees for compatibility with MongoDB. - That means latitude must be -90<=lat<=90 and longitude - must satisfy -180<=lat<=180. The function will try to handle the - common situation with 0<=lon<=360 by wrapping 90->180 values to - -180->0, A ValueError exception is thrown if + That means latitude must be -90<=lat<=90 and longitude + must satisfy -180<=lat<=180. The function will try to handle the + common situation with 0<=lon<=360 by wrapping 90->180 values to + -180->0, A ValueError exception is thrown if for any other situation with lot or lon outside those bounds. - - If you specify the optional "doc" argument it is assumed to be - a python dict to which the geoJSON point data is to be added. - By default a new dict is created that will contain only the - geoJSON point data. The doc options is useful if you want to - add the geoJSON data to the document before appending it. - The default is more useful for updates to add geospatial - query capabilities to a collection with lat-lon data that is + + If you specify the optional "doc" argument it is assumed to be + a python dict to which the geoJSON point data is to be added. + By default a new dict is created that will contain only the + geoJSON point data. The doc options is useful if you want to + add the geoJSON data to the document before appending it. + The default is more useful for updates to add geospatial + query capabilities to a collection with lat-lon data that is not properl structure. In all cases the geoJSON data is a - itself a python dict but a value associated accessible from + itself a python dict but a value associated accessible from the output dict with te key defined by the "key" argument - (default is 'epicenter', which is appropriate for earthquake + (default is 'epicenter', which is appropriate for earthquake source data.) - with a + with a """ outval = dict() - outval['type'] = 'Point' + outval["type"] = "Point" lon = float(lon) # make this it is a float for database consistency - if lon>180.0 and lon<=360.0: + if lon > 180.0 and lon <= 360.0: # we do this correction silently lon -= 360.0 lat = float(lat) if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0: message = "geoJSON_doc: Illegal geographic input\n" message += "latitude received={} MongoDB requires [-90,90] range\n".format(lat) - message += "longitude received={} MongoDB requires [-180,180] range".format(lon) + message += "longitude received={} MongoDB requires [-180,180] range".format( + lon + ) raise ValueError(message) - - outval['coordinates'] = [float(lon), float(lat)] + + outval["coordinates"] = [float(lon), float(lat)] if doc: retdoc = doc retdoc[key] = outval else: - retdoc={key : outval} + retdoc = {key: outval} return retdoc diff --git a/python/tests/db/test_database.py b/python/tests/db/test_database.py index 9dd73fdb9..23ce2fc73 100644 --- a/python/tests/db/test_database.py +++ b/python/tests/db/test_database.py @@ -49,7 +49,7 @@ from mspasspy.util import logging_helper from bson.objectid import ObjectId from datetime import datetime -from mspasspy.db.database import Database,geoJSON_doc +from mspasspy.db.database import Database, geoJSON_doc from mspasspy.db.client import DBClient @@ -3451,51 +3451,49 @@ def test_set_schema(self): self.db.set_schema("mspass_lite.yaml") with pytest.raises(KeyError, match="site"): self.db.database_schema._attr_dict["site"] - + def test_geoJSON_doc(self): """ - Tests only the function geoJSON_doc added Jan 2024 to properly - create geoJSON records that allow geospatial queries in - site, channel, and source. That function is now used in - Database.save_inventory and Database.save_catalog to always - save a geoJSOn format location data. We assume the bulk of the - code is already tested when those two functions are tested in - other test functions. The big thing here is testing the + Tests only the function geoJSON_doc added Jan 2024 to properly + create geoJSON records that allow geospatial queries in + site, channel, and source. That function is now used in + Database.save_inventory and Database.save_catalog to always + save a geoJSOn format location data. We assume the bulk of the + code is already tested when those two functions are tested in + other test functions. The big thing here is testing the bad data input handlers. """ # test null doc input - doc = geoJSON_doc(22.0,44.0,key='testpoint') - assert 'testpoint' in doc - val = doc['testpoint'] - assert val['type'] == "Point" - coords = val['coordinates'] + doc = geoJSON_doc(22.0, 44.0, key="testpoint") + assert "testpoint" in doc + val = doc["testpoint"] + assert val["type"] == "Point" + coords = val["coordinates"] # note coordinates pair is lon,lat assert coords[0] == 44.0 assert coords[1] == 22.0 - doc = geoJSON_doc(33.0,55.0,doc=doc,key='testpoint2') + doc = geoJSON_doc(33.0, 55.0, doc=doc, key="testpoint2") # doc should contain both testpoint and testpoint2 # This tests update feature of this function - assert 'testpoint' in doc - assert 'testpoint2' in doc - val = doc['testpoint2'] - assert val['type'] == "Point" - coords = val['coordinates'] + assert "testpoint" in doc + assert "testpoint2" in doc + val = doc["testpoint2"] + assert val["type"] == "Point" + coords = val["coordinates"] # note coordinates pair is lon,lat assert coords[0] == 55.0 assert coords[1] == 33.0 # recoverable value test - doc = geoJSON_doc(20,270.0,key='recoverable') - assert 'recoverable' in doc - val = doc['recoverable'] - coords = val['coordinates'] - # use is_close because -90 is computed but normal floating point math + doc = geoJSON_doc(20, 270.0, key="recoverable") + assert "recoverable" in doc + val = doc["recoverable"] + coords = val["coordinates"] + # use is_close because -90 is computed but normal floating point math # for that simple calculation would allow an == to also work - assert np.isclose(coords[0],-90.0) + assert np.isclose(coords[0], -90.0) assert coords[1] == 20.0 - - with pytest.raises(ValueError, - match="Illegal geographic input"): - doc = geoJSON_doc(20,400) + with pytest.raises(ValueError, match="Illegal geographic input"): + doc = geoJSON_doc(20, 400) From 81443ead016e68753e749ea1f50916f28eacaf4c Mon Sep 17 00:00:00 2001 From: wangyinz Date: Thu, 18 Jan 2024 08:26:58 -0600 Subject: [PATCH 6/7] replace starttime by calib in testing exclude_keys --- python/tests/db/test_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/db/test_database.py b/python/tests/db/test_database.py index 23ce2fc73..a6d526448 100644 --- a/python/tests/db/test_database.py +++ b/python/tests/db/test_database.py @@ -2128,13 +2128,13 @@ def test_check_undefined_keys(self): ts, mode="promiscuous", storage_mode="gridfs", - exclude_keys=["extra2", "starttime"], + exclude_keys=["extra2", "calib"], return_data=True, ) assert save_res.live self.db["wf_TimeSeries"].update_one({"_id": ts["_id"]}, {"$set": {"t0": 1.0}}) res = self.db["wf_TimeSeries"].find_one({"_id": ts["_id"]}) - assert "starttime" not in res + assert "calib" not in res assert "npts" not in res assert "t0" in res From 0393d02a9f130c073f9e287c943cf4b294bb54a0 Mon Sep 17 00:00:00 2001 From: wangyinz Date: Thu, 18 Jan 2024 09:06:56 -0600 Subject: [PATCH 7/7] now supports exclude_keys for starttime and endtime --- python/mspasspy/db/database.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/mspasspy/db/database.py b/python/mspasspy/db/database.py index 70d764880..92d2bae8c 100755 --- a/python/mspasspy/db/database.py +++ b/python/mspasspy/db/database.py @@ -6505,8 +6505,10 @@ def _atomic_save_all_documents( mspass_object.kill() # Always set starttime and endtime - insertion_dict["starttime"] = mspass_object.t0 - insertion_dict["endtime"] = mspass_object.endtime() + if not exclude_keys or "starttime" not in exclude_keys: + insertion_dict["starttime"] = mspass_object.t0 + if not exclude_keys or "endtime" not in exclude_keys: + insertion_dict["endtime"] = mspass_object.endtime() # add tag - intentionally not set in mspass_object returned if data_tag: