Debug thetadata feature and made it to work 1st time

lumibot/backtesting/backtesting_broker.py

@@ -638,6 +638,8 @@ def process_pending_orders(self, strategy):
                 low = df["low"].iloc[0]
                 close = df["close"].iloc[0]
                 volume = df["volume"].iloc[0]
+                print(f"in BacktestingBroker, asset: {asset}")
+                print(f"in BacktestingBroker, OHLCV: {open}, {high}, {low}, {close}, {volume}")
 
             #############################
             # Determine transaction price.

lumibot/backtesting/polygon_backtesting.py

@@ -78,7 +78,6 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
         start_datetime, ts_unit = self.get_start_datetime_and_ts_unit(
             length, timestep, start_dt, start_buffer=START_BUFFER
         )
-
         # Check if we have data for this asset
         if search_asset in self.pandas_data:
             asset_data = self.pandas_data[search_asset]
@@ -125,6 +124,7 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
         # Download data from Polygon
         try:
             # Get data from Polygon
+            date_time_now = self.get_datetime()
             df = polygon_helper.get_price_data_from_polygon(
                 self._api_key,
                 asset_separated,
@@ -133,6 +133,7 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                 timespan=ts_unit,
                 quote_asset=quote_asset,
             )
+            # df.to_csv(f"{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}_polygon.csv")
         except BadResponse as e:
             # Assuming e.message or similar attribute contains the error message
             formatted_start_datetime = start_datetime.strftime("%Y-%m-%d")
@@ -191,10 +192,11 @@ def _pull_source_symbol_bars(
     ):
         # Get the current datetime and calculate the start datetime
         current_dt = self.get_datetime()
-
+        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars current_dt:{current_dt}\n")
         # Get data from Polygon
+        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars calls self._update_pandas_data\n")
         self._update_pandas_data(asset, quote, length, timestep, current_dt)
-
+        print(f"\npolygon_backtesting.py:_pull_source_symbol_bars calls super()._pull_source_symbol_bars\n")
         return super()._pull_source_symbol_bars(
             asset, length, timestep, timeshift, quote, exchange, include_after_hours
         )

lumibot/backtesting/thetadata_backtesting.py

@@ -129,6 +129,7 @@ def update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
         # Download data from Polygon
         try:
             # Get data from Polygon
+            date_time_now = self.get_datetime()
             df = thetadata_helper.get_price_data(
                 self._username,
                 self._password,
@@ -137,9 +138,12 @@ def update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
                 self.datetime_end,
                 timespan=ts_unit,
                 quote_asset=quote_asset,
+                dt=self.get_datetime()
             )
+            # save df to csv file
+            df.to_csv(f"{date_time_now}_{asset.strike}_{asset.expiration}_{asset.right}.csv")
         except Exception as e:
-            logging.error(traceback.format_exc())
+            logging.info(traceback.format_exc())
             raise Exception("Error getting data from ThetaData") from e
 
         if df is None:

lumibot/data_sources/pandas_data.py

@@ -248,6 +248,7 @@ def _pull_source_symbol_bars(
 
         now = self.get_datetime()
         try:
+            print(f"\npandas_data.py:_pull_source_symbol_bars calls data.get_bars, will select 2 bars from the whole data table\n")
             res = data.get_bars(now, length=length, timestep=timestep, timeshift=timeshift)
         # Return None if data.get_bars returns a ValueError
         except ValueError as e:
@@ -413,7 +414,7 @@ def get_historical_prices(
 
         if not timestep:
             timestep = self.get_timestep()
-
+        print(f"\npandas_data.py:get_historial prices calls self._pull_source_symbol_bars\n")
         response = self._pull_source_symbol_bars(
             asset,
             length,

lumibot/strategies/_strategy.py

@@ -466,7 +466,7 @@ def _update_portfolio_value(self):
                 prices = self.broker.option_source.get_last_prices(assets)
             else:
                 prices = self.broker.data_source.get_last_prices(assets)
-
+            print(f"\n_strategy.py datetime:{self.broker.datetime} prices: {prices}\n")
             for position in positions:
                 # Turn the asset into a tuple if it's a crypto asset
                 asset = (
@@ -515,7 +515,9 @@ def _update_portfolio_value(self):
                 else:
                     multiplier = asset.multiplier if asset.asset_type in ["option", "future"] else 1
                 portfolio_value += float(quantity) * price * multiplier
-
+                print(f"quantity: {quantity}, price: {price}, multiplier: {multiplier}")
+                print(f"portfolio_value += {float(quantity)} * {price} * {multiplier}")
+            print(f"portfolio_value: {portfolio_value}")
             self._portfolio_value = portfolio_value
 
         return portfolio_value

lumibot/strategies/strategy.py

@@ -3088,9 +3088,10 @@ def get_historical_prices(
         asset = self.crypto_assets_to_tuple(asset, quote)
         if not timestep:
             timestep = self.broker.data_source.MIN_TIMESTEP
-        print(f"\nget_historial prices: time: {self.broker.datetime}")
+        print(f"\nstrategy.py:get_historial prices: time: {self.broker.datetime}")
         if self.broker.option_source and asset.asset_type == "option":
-            print(f"\nSTRATEGY: Found option source in broker, and asset type is 'option', expiry:{asset.expiration}, strike:{asset.strike}, right:{asset.right}")
+            print(
+                f"\nSTRATEGY: Found option source in broker, and asset type is 'option', expiry:{asset.expiration}, strike:{asset.strike}, right:{asset.right}")
             return self.broker.option_source.get_historical_prices(
                 asset,
                 length,

lumibot/tools/thetadata_helper.py

@@ -15,6 +15,7 @@
 
 WAIT_TIME = 60
 MAX_DAYS = 30
+THETA_TIME_SHIFT = 4
 CACHE_SUBFOLDER = "thetadata"
 BASE_URL = "http://127.0.0.1:25510"
 
@@ -27,6 +28,7 @@ def get_price_data(
     end: datetime,
     timespan: str = "minute",
     quote_asset: Asset = None,
+    dt=None
 ):
     """
     Queries ThetaData for pricing data for the given asset and returns a DataFrame with the data. Data will be
@@ -57,7 +59,8 @@ def get_price_data(
         A DataFrame with the pricing data for the asset
 
     """
-
+    if start.date() < dt.date():
+        start = dt
     # Check if we already have data for this asset in the feather file
     df_all = None
     df_feather = None
@@ -71,6 +74,7 @@ def get_price_data(
     # Check if we need to get more data
     missing_dates = get_missing_dates(df_all, asset, start, end)
     if not missing_dates:
+        df_all.index = df_all.index + pd.Timedelta(hours=THETA_TIME_SHIFT)
         return df_all
 
     logging.info(
@@ -119,6 +123,7 @@ def get_price_data(
             break
 
     update_cache(cache_file, df_all, df_feather)
+    df_all.index = df_all.index + pd.Timedelta(hours=THETA_TIME_SHIFT)
     return df_all
 
 
@@ -277,20 +282,25 @@ def update_df(df_all, result):
         df = df.set_index("datetime").sort_index()
         if df_all is not None:
             print(f"\n new loop df_all head: \n{df_all.head()}")
+            # set "datetime" column as index of df_all
+            df_all = df_all.set_index("datetime").sort_index()
+            print(f"\n after setting index df_all head: \n{df_all.head()}")
         print(f"\nbefore utc: df head: \n{df.head()}")
         df.index = df.index.tz_localize("UTC")
         print(f"\nafter utc: df head: \n{df.head()}")
         if df_all is None or df_all.empty:
             df_all = df
+            print(f"\nNo df_all, assign df_all=df")
         else:
             print(f"\nInside update_df, df_all head: \n{df_all.head()}")
             print(f"\nInside update_df, df head: \n{df.head()}")
             df_all = pd.concat([df_all, df]).sort_index()
             df_all = df_all[~df_all.index.duplicated(keep="first")]  # Remove any duplicate rows
 
-        print(f"\nafter concat df_all head: \n{df_all.head()}")
-        df_all = df_all.reset_index()
+            print(f"\nafter concat df_all head: \n{df_all.head()}")
+        # df_all = df_all.reset_index()
         print(f"\nafter concat reset index df_all head: \n{df_all.head()}")
+
     return df_all
 
 
@@ -391,9 +401,9 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
 
                 # Check if json_resp has error_type inside of header
                 if "error_type" in json_resp["header"] and json_resp["header"]["error_type"] != "null":
-                    logging.error(f"Error getting data from Theta Data: {json_resp['header']['error_type']}")
+                    logging.error(
+                        f"Error getting data from Theta Data: {json_resp['header']['error_type']},\nquerystring: {querystring}")
                     check_connection(username=username, password=password)
-                    print(f"\nthe_helper: Cannot find valid querystring: {querystring}")
                 else:
                     print(f"\nthe_helper: Found valid querystring: {querystring}")
                     break
@@ -402,7 +412,7 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
             check_connection(username=username, password=password)
 
         counter += 1
-        if counter > 3:
+        if counter > 1:
             raise ValueError("Cannot connect to Theta Data!")
 
     return json_resp