Fix bug in handling starttime and endtime

data/yaml/mspass.yaml

@@ -66,6 +66,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional 
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

data/yaml/mspass_fdsn.yaml

@@ -62,6 +62,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

data/yaml/mspass_lite.yaml

@@ -37,6 +37,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.
@@ -283,4 +287,5 @@ Metadata:
         readonly: false
       loc:
         collection: wf_Seismogram
-        readonly: false 
+        readonly: false
+

data/yaml/mspass_s3.yaml

@@ -62,6 +62,10 @@ Database:
           concept:  Time shift applied to define relative time of 0.0
           aliases:  t0shift
           constraint: required
+        endtime:
+          type: double
+          concept: Time of last sample of data (epoch time or relative to some other time mark)
+          constraint: optional
         utc_convertible:
           type:  bool
           concept:  When true starttime_shift can be used to convert relative time to UTC.

python/mspasspy/db/database.py

@@ -1,4 +1,3 @@
-
 import os
 import io
 import copy
@@ -4960,13 +4959,13 @@
                     loc_etime = self._handle_null_endtime(loc_etime)
                     rec["lat"] = loc_lat
                     rec["lon"] = loc_lon
-                    # save coordinates in both geoJSON and "legacy" 
-                    # format 
-                    rec["coords"] = [loc_lon,loc_lat]
-                    # Illegal lon,lat values will cause this to throw a 
-                    # ValueError exception.   We let it do that as 
+                    # save coordinates in both geoJSON and "legacy"
+                    # format
+                    rec["coords"] = [loc_lon, loc_lat]
+                    # Illegal lon,lat values will cause this to throw a
+                    # ValueError exception.   We let it do that as
                     # it indicates a problem datum
-                    rec = geoJSON_doc(loc_lat,loc_lon,doc=rec,key='location')
+                    rec = geoJSON_doc(loc_lat, loc_lon, doc=rec, key="location")
                     rec["elev"] = loc_elev
                     rec["edepth"] = loc_edepth
                     rec["starttime"] = starttime.timestamp
@@ -5493,12 +5492,12 @@
             rec["lat"] = o.latitude
             rec["lon"] = o.longitude
             # save the epicenter data in both legacy format an d
-            # geoJSON format.  Either can technically be used in a 
-            # geospatial query but the geoJSON version is always 
+            # geoJSON format.  Either can technically be used in a
+            # geospatial query but the geoJSON version is always
             # preferred
             rec["coords"] = [o.longitude, o.latitude]
             # note this function updates rec and returns the upodate
-            rec = geoJSON_doc(o.latitude, o.longitude, doc=rec,key='epicenter')
+            rec = geoJSON_doc(o.latitude, o.longitude, doc=rec, key="epicenter")
             # It appears quakeml puts source depths in meter
             # convert to km
             # also obspy's catalog object seesm to allow depth to be
@@ -6505,6 +6504,12 @@
             # Warning:  this can lead to sample data save orphans
             mspass_object.kill()
 
+        # Always set starttime and endtime
+        if not exclude_keys or "starttime" not in exclude_keys:
+            insertion_dict["starttime"] = mspass_object.t0
+        if not exclude_keys or "endtime" not in exclude_keys:
+            insertion_dict["endtime"] = mspass_object.endtime()
+
         # add tag - intentionally not set in mspass_object returned
         if data_tag:
             insertion_dict["data_tag"] = data_tag
@@ -8186,54 +8191,57 @@
                     mdout.erase(k)
     return mdout
 
-def geoJSON_doc(lat,lon,doc=None,key='epicenter')->dict:
+
+def geoJSON_doc(lat, lon, doc=None, key="epicenter") -> dict:
     """
     Convenience function to create a geoJSON format point object document
-    from a points specified by latitude and longitude. The format 
-    for a geoJSON point isn't that strange but how to structure it into 
+    from a points specified by latitude and longitude. The format
+    for a geoJSON point isn't that strange but how to structure it into
     a mongoDB document for use with geospatial queries is not as
-    clear from current MongoDB documentation.  This function makes that 
+    clear from current MongoDB documentation.  This function makes that
     proess easier.
-    
-    The required inpput is latitude (lat) and longitude (lon).  The 
+
+    The required inpput is latitude (lat) and longitude (lon).  The
     values are assumed to be in degrees for compatibility with MongoDB.
-    That means latitude must be -90<=lat<=90 and longitude 
-    must satisfy -180<=lat<=180.  The function will try to handle the 
-    common situation with 0<=lon<=360 by wrapping 90->180 values to 
-    -180->0,  A ValueError exception is thrown if  
+    That means latitude must be -90<=lat<=90 and longitude
+    must satisfy -180<=lat<=180.  The function will try to handle the
+    common situation with 0<=lon<=360 by wrapping 90->180 values to
+    -180->0,  A ValueError exception is thrown if
     for any other situation with lot or lon outside those bounds.
-    
-    If you specify the optional "doc" argument it is assumed to be 
-    a python dict to which the geoJSON point data is to be added.  
-    By default a new dict is created that will contain only the 
-    geoJSON point data.  The doc options is useful if you want to 
-    add the geoJSON data to the document before appending it.  
-    The default is more useful for updates to add geospatial 
-    query capabilities to a collection with lat-lon data that is 
+
+    If you specify the optional "doc" argument it is assumed to be
+    a python dict to which the geoJSON point data is to be added.
+    By default a new dict is created that will contain only the
+    geoJSON point data.  The doc options is useful if you want to
+    add the geoJSON data to the document before appending it.
+    The default is more useful for updates to add geospatial
+    query capabilities to a collection with lat-lon data that is
     not properl structure.  In all cases the geoJSON data is a
-    itself a python dict but a value associated accessible from 
+    itself a python dict but a value associated accessible from
     the output dict with te key defined by the "key" argument
-    (default is 'epicenter', which is appropriate for earthquake 
+    (default is 'epicenter', which is appropriate for earthquake
     source data.)
-    with a 
+    with a
     """
     outval = dict()
-    outval['type'] = 'Point'
+    outval["type"] = "Point"
     lon = float(lon)  # make this it is a float for database consistency
-    if lon>180.0 and lon<=360.0:
+    if lon > 180.0 and lon <= 360.0:
         # we do this correction silently
         lon -= 360.0
     lat = float(lat)
     if lat < -90.0 or lat > 90.0 or lon < -180.0 or lon > 180.0:
         message = "geoJSON_doc:  Illegal geographic input\n"
         message += "latitude received={}  MongoDB requires [-90,90] range\n".format(lat)
-        message += "longitude received={}  MongoDB requires [-180,180] range".format(lon)
+        message += "longitude received={}  MongoDB requires [-180,180] range".format(
+            lon
+        )
         raise ValueError(message)
-        
-    outval['coordinates'] = [float(lon), float(lat)]
+
+    outval["coordinates"] = [float(lon), float(lat)]
     if doc:
         retdoc = doc
         retdoc[key] = outval
     else:
-        retdoc={key : outval}
+        retdoc = {key: outval}
     return retdoc

python/tests/db/test_database.py

@@ -49,7 +49,7 @@
 from mspasspy.util import logging_helper
 from bson.objectid import ObjectId
 from datetime import datetime
-from mspasspy.db.database import Database,geoJSON_doc
+from mspasspy.db.database import Database, geoJSON_doc
 from mspasspy.db.client import DBClient
 
 
@@ -2128,13 +2128,13 @@ def test_check_undefined_keys(self):
             ts,
             mode="promiscuous",
             storage_mode="gridfs",
-            exclude_keys=["extra2", "starttime"],
+            exclude_keys=["extra2", "calib"],
             return_data=True,
         )
         assert save_res.live
         self.db["wf_TimeSeries"].update_one({"_id": ts["_id"]}, {"$set": {"t0": 1.0}})
         res = self.db["wf_TimeSeries"].find_one({"_id": ts["_id"]})
-        assert "starttime" not in res
+        assert "calib" not in res
         assert "npts" not in res
         assert "t0" in res
 
@@ -3451,51 +3451,49 @@ def test_set_schema(self):
         self.db.set_schema("mspass_lite.yaml")
         with pytest.raises(KeyError, match="site"):
             self.db.database_schema._attr_dict["site"]
-            
+
     def test_geoJSON_doc(self):
         """
-        Tests only the function geoJSON_doc added Jan 2024 to properly 
-        create geoJSON records that allow geospatial queries in 
-        site, channel, and source.  That function is now used in 
-        Database.save_inventory and Database.save_catalog to always 
-        save a geoJSOn format location data.   We assume the bulk of the 
-        code is already tested when those two functions are tested in 
-        other test functions.  The big thing here is testing the 
+        Tests only the function geoJSON_doc added Jan 2024 to properly
+        create geoJSON records that allow geospatial queries in
+        site, channel, and source.  That function is now used in
+        Database.save_inventory and Database.save_catalog to always
+        save a geoJSOn format location data.   We assume the bulk of the
+        code is already tested when those two functions are tested in
+        other test functions.  The big thing here is testing the
         bad data input handlers.
         """
         # test null doc input
-        doc = geoJSON_doc(22.0,44.0,key='testpoint')
-        assert 'testpoint' in doc
-        val = doc['testpoint']
-        assert val['type'] == "Point"
-        coords = val['coordinates']
+        doc = geoJSON_doc(22.0, 44.0, key="testpoint")
+        assert "testpoint" in doc
+        val = doc["testpoint"]
+        assert val["type"] == "Point"
+        coords = val["coordinates"]
         # note coordinates pair is lon,lat
         assert coords[0] == 44.0
         assert coords[1] == 22.0
 
-        doc = geoJSON_doc(33.0,55.0,doc=doc,key='testpoint2')
+        doc = geoJSON_doc(33.0, 55.0, doc=doc, key="testpoint2")
         #  doc should contain both testpoint and testpoint2
         #  This tests update feature of this function
-        assert 'testpoint' in doc
-        assert 'testpoint2' in doc
-        val = doc['testpoint2']
-        assert val['type'] == "Point"
-        coords = val['coordinates']
+        assert "testpoint" in doc
+        assert "testpoint2" in doc
+        val = doc["testpoint2"]
+        assert val["type"] == "Point"
+        coords = val["coordinates"]
         # note coordinates pair is lon,lat
         assert coords[0] == 55.0
         assert coords[1] == 33.0
 
         # recoverable value test
-        doc = geoJSON_doc(20,270.0,key='recoverable')
-        assert 'recoverable' in doc
-        val = doc['recoverable']
-        coords = val['coordinates']
-        # use is_close because -90 is computed but normal floating point math 
+        doc = geoJSON_doc(20, 270.0, key="recoverable")
+        assert "recoverable" in doc
+        val = doc["recoverable"]
+        coords = val["coordinates"]
+        # use is_close because -90 is computed but normal floating point math
         # for that simple calculation would allow an == to also work
-        assert np.isclose(coords[0],-90.0)
+        assert np.isclose(coords[0], -90.0)
         assert coords[1] == 20.0
-
-        with pytest.raises(ValueError, 
-                           match="Illegal geographic input"):
-            doc = geoJSON_doc(20,400)
 
+        with pytest.raises(ValueError, match="Illegal geographic input"):
+            doc = geoJSON_doc(20, 400)