From 3397f5fea2691bdb4080341818e00cb19788ca0d Mon Sep 17 00:00:00 2001 From: James McManus Date: Wed, 3 Jul 2024 14:58:48 -0400 Subject: [PATCH 1/5] Iin case where therer is no forecast data added 120 hours (5 days) to end_date so more tidal predictions data is shown --- src/common/pg_impl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/pg_impl.py b/src/common/pg_impl.py index 9f9d186..f616627 100644 --- a/src/common/pg_impl.py +++ b/src/common/pg_impl.py @@ -298,7 +298,9 @@ def get_station_data(self, **kwargs) -> str: # check for an error if forecast_data.empty: - end_date = kwargs['time_mark'] + # If no forecast data add 120 hours (5 days) to end_date for the tidal predictions data + #end_date = kwargs['time_mark'] + end_date = (datetime.fromisoformat(kwargs['time_mark']) + timedelta(5)).isoformat() else: # get end_date from last datetime in forecast data end_date = forecast_data['time_stamp'].iloc[-1] From f45d946d3ea5570f6b3cc16a38810f8fad38553a Mon Sep 17 00:00:00 2001 From: James McManus Date: Mon, 8 Jul 2024 15:53:01 -0400 Subject: [PATCH 2/5] Removed T from timemark when running obs_data.loc[obs_data.time_stamp >= timemark, col] = np.nan --- src/common/pg_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/pg_impl.py b/src/common/pg_impl.py index f616627..31901aa 100644 --- a/src/common/pg_impl.py +++ b/src/common/pg_impl.py @@ -367,7 +367,8 @@ def get_station_data(self, **kwargs) -> str: # convert all values after the time mark to nan, in obs data, except in the time_stamp and tidal_predictions columns for col in obs_data.columns: if col not in ('time_stamp', 'tidal_predictions'): - obs_data.loc[obs_data.time_stamp >= kwargs['time_mark'], col] = np.nan + timemark = " ".join(kwargs['time_mark'].split('T')) + obs_data.loc[obs_data.time_stamp >= timemark, col] = np.nan else: continue From 94eed7bfc837bb337e5ae497a9204e1e7d6b11a4 Mon Sep 17 00:00:00 2001 From: James McManus Date: Mon, 15 Jul 2024 08:11:22 -0400 Subject: [PATCH 3/5] Added tidal_gauge_water_level as column to not trunkate on timemark --- src/common/pg_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/pg_impl.py b/src/common/pg_impl.py index 31901aa..803fc4a 100644 --- a/src/common/pg_impl.py +++ b/src/common/pg_impl.py @@ -366,7 +366,7 @@ def get_station_data(self, **kwargs) -> str: # convert all values after the time mark to nan, in obs data, except in the time_stamp and tidal_predictions columns for col in obs_data.columns: - if col not in ('time_stamp', 'tidal_predictions'): + if col not in ('time_stamp', 'tidal_predictions', 'tidal_gauge_water_level'): timemark = " ".join(kwargs['time_mark'].split('T')) obs_data.loc[obs_data.time_stamp >= timemark, col] = np.nan else: From c518f103f3a36cab39d58a9b305dafaf98a8499a Mon Sep 17 00:00:00 2001 From: James McManus Date: Fri, 19 Jul 2024 10:22:27 -0400 Subject: [PATCH 4/5] Added max_forecast_endtime to get_station_data() and get_forecast_station_data() --- src/common/pg_impl.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/common/pg_impl.py b/src/common/pg_impl.py index 803fc4a..af760e7 100644 --- a/src/common/pg_impl.py +++ b/src/common/pg_impl.py @@ -290,8 +290,11 @@ def get_station_data(self, **kwargs) -> str: :param kwargs: :return: """ + # calculate max_forecast_endtime from time_mark + max_forecast_endtime = (datetime.fromisoformat(kwargs['time_mark']) + timedelta(14)).isoformat() + # get forecast data - forecast_data = self.get_forecast_station_data(kwargs['station_name'], kwargs['time_mark'], kwargs['data_source'], kwargs['instance_name']) + forecast_data = self.get_forecast_station_data(kwargs['station_name'], kwargs['time_mark'], max_forecast_endtime, kwargs['data_source'], kwargs['instance_name']) # derive start date from the time mark start_date = (datetime.fromisoformat(kwargs['time_mark']) - timedelta(4)).isoformat() @@ -299,7 +302,6 @@ def get_station_data(self, **kwargs) -> str: # check for an error if forecast_data.empty: # If no forecast data add 120 hours (5 days) to end_date for the tidal predictions data - #end_date = kwargs['time_mark'] end_date = (datetime.fromisoformat(kwargs['time_mark']) + timedelta(5)).isoformat() else: # get end_date from last datetime in forecast data @@ -411,12 +413,13 @@ def get_station_data(self, **kwargs) -> str: # return the data to the caller return station_df.to_csv(index=False) - def get_forecast_station_data(self, station_name, time_mark, data_source, instance_name) -> pd.DataFrame: + def get_forecast_station_data(self, station_name, time_mark, max_forecast_endtime, data_source, instance_name) -> pd.DataFrame: """ Gets the forcast station data :param station_name: :param time_mark: + :param max_forecast_endtime: :param data_source: :param instance_name: :return: @@ -426,7 +429,7 @@ def get_forecast_station_data(self, station_name, time_mark, data_source, instan # Run query sql = f"SELECT * FROM get_forecast_timeseries_station_data(_station_name := '{station_name}', _timemark := '{time_mark}', " \ - f"_data_source := '{data_source}', _source_instance := '{instance_name}')" + f"_max_forecast_endtime := '{max_forecast_endtime}', _data_source := '{data_source}', _source_instance := '{instance_name}')" # get the info station_data = self.exec_sql('apsviz_gauges', sql) From 70487928f57714edc0664696dd510216f84a8796 Mon Sep 17 00:00:00 2001 From: James McManus Date: Mon, 22 Jul 2024 09:15:29 -0400 Subject: [PATCH 5/5] Added fix when no tidal observation data, but tidal prediction exists --- src/common/pg_impl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/pg_impl.py b/src/common/pg_impl.py index af760e7..75a15ab 100644 --- a/src/common/pg_impl.py +++ b/src/common/pg_impl.py @@ -335,8 +335,10 @@ def get_station_data(self, **kwargs) -> str: # check if obs_data.columns id buoy. If it is search for wave_height, else search for water_level if 'ocean_buoy_wave_height' in obs_data.columns: observation_name = [s for s in obs_data.columns.values if 'wave_height' in s][0] - else: + elif 'water_level' in obs_data.columns: observation_name = [s for s in obs_data.columns.values if 'water_level' in s][0] + else: + observation_name = None # Merge nowcast_data with obs_data obs_data = obs_data.merge(nowcast_data, on='time_stamp', how='outer')