Debugged thetadata feature, added summer vs winter timezone adjustmen…

…t, subtracted 1 min from thetadata

lumibot/backtesting/backtesting_broker.py

@@ -93,10 +93,14 @@ def _update_datetime(self, update_dt, cash=None, portfolio_value=None):
             new_datetime = self.datetime + timedelta(seconds=update_dt)
         else:
             new_datetime = update_dt
-
+        print(f"\n backtesting_broker.py _update_datetime, new_datetime {new_datetime}")
         self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
+        print(
+            f"\n backtesting_broker.py _update_datetime, self.data_source.get_datetime() {self.data_source.get_datetime()}")
         if self.option_source:
             self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
+            print(
+                f"\n backtesting_broker.py _update_datetime, self.option_source.get_datetime() {self.option_source.get_datetime()}")
         logging.info(f"Current backtesting datetime {self.datetime}")
 
     # =========Clock functions=====================

lumibot/backtesting/polygon_backtesting.py

@@ -125,6 +125,7 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
         try:
             # Get data from Polygon
             date_time_now = self.get_datetime()
+            # print(f"\npolygon_backtesting.py:before extracting data from helper: time:{date_time_now}\n")
             df = polygon_helper.get_price_data_from_polygon(
                 self._api_key,
                 asset_separated,
@@ -133,7 +134,7 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                 timespan=ts_unit,
                 quote_asset=quote_asset,
             )
-            # df.to_csv(f"{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}_polygon.csv")
+            # df.to_csv(f"polygon_csv/{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}_polygon.csv")
         except BadResponse as e:
             # Assuming e.message or similar attribute contains the error message
             formatted_start_datetime = start_datetime.strftime("%Y-%m-%d")
@@ -171,12 +172,14 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
 
         if (df is None) or df.empty:
             return
-
+        # print(f"\npolygon_backtesting.py:after extracting data from helper: time:{self.get_datetime()}\n")
         data = Data(asset_separated, df, timestep=ts_unit, quote=quote_asset)
+        # print(f"\npolygon_backtesting.py:after data packing: time:{self.get_datetime()}\n")
         pandas_data_update = self._set_pandas_data_keys([data])
-
+        # print(f"\npolygon_backtesting.py:after update keys: time:{self.get_datetime()}\n")
         # Add the keys to the self.pandas_data dictionary
         self.pandas_data.update(pandas_data_update)
+        # print(f"\npolygon_backtesting.py:after update data: time:{self.get_datetime()}\n")
         if PolygonDataBacktesting.MAX_STORAGE_BYTES:
             self._enforce_storage_limit(self.pandas_data)
 
@@ -192,11 +195,11 @@ def _pull_source_symbol_bars(
     ):
         # Get the current datetime and calculate the start datetime
         current_dt = self.get_datetime()
-        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars current_dt:{current_dt}\n")
+        # print(f"\npolygon_backtesting.py:_pull_source_symbol_bars current_dt:{current_dt}\n")
         # Get data from Polygon
-        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars calls self._update_pandas_data\n")
         self._update_pandas_data(asset, quote, length, timestep, current_dt)
-        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars calls super()._pull_source_symbol_bars\n")
+        # print(
+        #     f"\npolygon_backtesting.py:_pull_source_symbol_bars calls super()._pull_source_symbol_bars,current_dt:{current_dt}\n")
         return super()._pull_source_symbol_bars(
             asset, length, timestep, timeshift, quote, exchange, include_after_hours
         )

lumibot/backtesting/thetadata_backtesting.py

@@ -13,7 +13,7 @@
 
 class ThetaDataBacktesting(PandasData):
     """
-    Backtesting implementation of Polygon
+    Backtesting implementation of ThetaData
     """
 
     def __init__(
@@ -126,9 +126,9 @@ def update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                         # We don't have enough data, so we need to get more (but in minutes)
                         ts_unit = "minute"
 
-        # Download data from Polygon
+        # Download data from ThetaData
         try:
-            # Get data from Polygon
+            # Get data from ThetaData
             date_time_now = self.get_datetime()
             df = thetadata_helper.get_price_data(
                 self._username,
@@ -141,7 +141,7 @@ def update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                 dt=self.get_datetime()
             )
             # save df to csv file
-            df.to_csv(f"{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}.csv")
+            # df.to_csv(f"theta_csv/wrong{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}.csv")
         except Exception as e:
             logging.info(traceback.format_exc())
             raise Exception("Error getting data from ThetaData") from e
@@ -217,7 +217,7 @@ def get_last_price(self, asset, timestep="minute", quote=None, exchange=None, **
 
     def get_chains(self, asset):
         """
-        Integrates the Polygon client library into the LumiBot backtest for Options Data in the same
+        Integrates the ThetaData client library into the LumiBot backtest for Options Data in the same
         structure as Interactive Brokers options chain data
 
         Parameters

lumibot/data_sources/data_source_backtesting.py

@@ -71,6 +71,7 @@ def get_datetime_range(self, length, timestep="minute", timeshift=None):
         return start_date, end_date
 
     def _update_datetime(self, new_datetime, cash=None, portfolio_value=None):
+        print("Updating datetime to", new_datetime)
         self._datetime = new_datetime
         print_progress_bar(
             new_datetime,

lumibot/data_sources/pandas_data.py

@@ -202,8 +202,10 @@ def get_last_price(self, asset, quote=None, exchange=None):
 
     def get_last_prices(self, assets, quote=None, exchange=None, **kwargs):
         result = {}
+        print(f"\nPandasData get_last_prices, before results, time: {self.get_datetime()}")
         for asset in assets:
             result[asset] = self.get_last_price(asset, quote=quote, exchange=exchange)
+            print(f"\nPandasData get_last_prices after results, time: {self.get_datetime()}")
         return result
 
     def find_asset_in_data_store(self, asset, quote=None):
@@ -248,8 +250,9 @@ def _pull_source_symbol_bars(
 
         now = self.get_datetime()
         try:
-            print(f"\npandas_data.py:_pull_source_symbol_bars calls data.get_bars, will select 2 bars from the whole data table\n")
+            print(f"\npandas_data.py:_pull_source_symbol_bars calls data.get_bars, will select 2 bars, time:{now}")
             res = data.get_bars(now, length=length, timestep=timestep, timeshift=timeshift)
+            print(f"\npandas_data.py:_pull_source_symbol_bars after data.get_bars, time:{now}")
         # Return None if data.get_bars returns a ValueError
         except ValueError as e:
             logging.info(f"Error getting bars for {asset}: {e}")

lumibot/strategies/_strategy.py

@@ -459,7 +459,7 @@ def _update_portfolio_value(self):
                     elif asset.asset_type == "option":
                         asset_is_option = True
                     assets.append(asset)
-
+            print(f"\n_strategy.py before using broker to get_last_prices, broker time: {self.broker.datetime}\n")
             if self.broker.option_source and asset_is_option:
                 print(
                     f"\n_strategy.py Found option source in broker, and asset type is 'option', time: {self.broker.datetime}\n")

lumibot/strategies/strategy_executor.py

@@ -252,7 +252,7 @@ def process_event(self, event, payload):
                 self.strategy._update_cash(order.side, quantity, price, multiplier)
 
             self._on_partially_filled_order(**payload)
-        
+
         else:
             self.strategy.logger.error(f"Event {event} not recognized. Payload: {payload}")
 
@@ -791,9 +791,9 @@ def _run_trading_session(self):
                 self.broker.data_source._iter_count += 1
 
             dt = self.broker.data_source._date_index[self.broker.data_source._iter_count]
-
+            print(f"strategy_executor.py: _run_trading_session: before _update_datetime time: {self.broker.datetime}")
             self.broker._update_datetime(dt, cash=self.strategy.cash, portfolio_value=self.strategy.portfolio_value)
-
+            print(f"strategy_executor.py: _run_trading_session: after _update_datetime time: {self.broker.datetime}")
             self.strategy._update_cash_with_dividends()
 
             self._on_trading_iteration()
@@ -956,7 +956,7 @@ def run(self):
 
         # Sort the trading days by market close time so that we can search them faster
         self.broker._trading_days.sort_values('market_close', inplace=True)  # Ensure sorted order
-        
+
         # Set DataFrame index to market_close for fast lookups
         self.broker._trading_days.set_index('market_close', inplace=True)
 

lumibot/tools/thetadata_helper.py

@@ -15,11 +15,26 @@
 
 WAIT_TIME = 60
 MAX_DAYS = 30
-THETA_TIME_SHIFT = 4
+THETA_SUMMER_TIME_SHIFT = 4
+THETA_WINTER_TIME_SHIFT = 5
 CACHE_SUBFOLDER = "thetadata"
 BASE_URL = "http://127.0.0.1:25510"
 
 
+def is_summer_time(date_time):
+    month = int(date_time.month)
+    if month >= 4 and month <= 10:
+        return True
+    elif month in [12, 1, 2]:
+        return False
+    else:
+        # Convert the datetime object to a struct_time in the local timezone
+        time_tuple = date_time.timetuple()
+
+        # Use the tm_isdst attribute to determine DST status
+        return time.localtime(time.mktime(time_tuple)).tm_isdst > 0
+
+
 def get_price_data(
     username: str,
     password: str,
@@ -74,7 +89,13 @@ def get_price_data(
     # Check if we need to get more data
     missing_dates = get_missing_dates(df_all, asset, start, end)
     if not missing_dates:
-        df_all.index = df_all.index + pd.Timedelta(hours=THETA_TIME_SHIFT)
+        # df_all.index = df_all.index + pd.Timedelta(hours=THETA_SUMMER_TIME_SHIFT) - pd.Timedelta(minutes=1)
+        if is_summer_time(start):
+            print(f"Today's date is in Daylight Saving Time (Summer Time)")
+            df_all.index = df_all.index + pd.Timedelta(hours=THETA_SUMMER_TIME_SHIFT) - pd.Timedelta(minutes=1)
+        else:
+            print(f"Today's date is in Standard Time (Winter Time)")
+            df_all.index = df_all.index + pd.Timedelta(hours=THETA_WINTER_TIME_SHIFT) - pd.Timedelta(minutes=1)
         return df_all
 
     logging.info(
@@ -114,7 +135,7 @@ def get_price_data(
 
         else:
             df_all = update_df(df_all, result_df)
-            print(f"\ndf_all head: \n{df_all.head()}")
+            print(f"\ndf_all tail: \n{df_all.tail()}")
 
         start = end + timedelta(days=1)
         end = start + delta
@@ -123,7 +144,14 @@ def get_price_data(
             break
 
     update_cache(cache_file, df_all, df_feather)
-    df_all.index = df_all.index + pd.Timedelta(hours=THETA_TIME_SHIFT)
+    # df_all.index = df_all.index + pd.Timedelta(hours=THETA_SUMMER_TIME_SHIFT) - pd.Timedelta(minutes=1)
+
+    if is_summer_time(start):
+        print(f"Today's date is in Daylight Saving Time (Summer Time)")
+        df_all.index = df_all.index + pd.Timedelta(hours=THETA_SUMMER_TIME_SHIFT) - pd.Timedelta(minutes=1)
+    else:
+        print(f"Today's date is in Standard Time (Winter Time)")
+        df_all.index = df_all.index + pd.Timedelta(hours=THETA_WINTER_TIME_SHIFT) - pd.Timedelta(minutes=1)
     return df_all
 
 
@@ -405,7 +433,7 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
                         f"Error getting data from Theta Data: {json_resp['header']['error_type']},\nquerystring: {querystring}")
                     check_connection(username=username, password=password)
                 else:
-                    print(f"\nthe_helper: Found valid querystring: {querystring}")
+                    # print(f"\nthe_helper: Found valid querystring: {querystring}")
                     break
 
         except Exception as e: