Enhancements and Bug Fixes (#2)

* refactor: validation that user can subscribe on real time update

* fix: config for Rest Url to run client externally

* feat: handle WS's error response

* feat: add explicit attribute for test classes

* remove: not used configs

* feat: ReSubscription process when internet was disconnected

* fix:workflow: name of test dll

* refactor: OptionChainProvider list
refactor: GetOptionContracts in DataDownloader

* fix: QuoteBars with Resolution greater then Tick
feat: QuoteBar tests
revert: data-folder path in config

* refactor: name of variable

.github/workflows/gh-actions.yml

@@ -46,4 +46,4 @@ jobs:
         run: dotnet build ./QuantConnect.ThetaData.Tests/QuantConnect.DataSource.ThetaData.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
       - name: Run QuantConnect.ThetaData.Tests
-        run: dotnet test ./QuantConnect.ThetaData.Tests/bin/Release/QuantConnect.Lean.DataSource.ThetaData.dll
+        run: dotnet test ./QuantConnect.ThetaData.Tests/bin/Release/QuantConnect.Lean.DataSource.ThetaData.Tests.dll

QuantConnect.ThetaData.Tests/TestHelpers.cs

@@ -53,8 +53,11 @@ public static void ValidateHistoricalBaseData(IEnumerable<BaseData> history, Res
                 case TickType.Trade:
                     AssertTradeTickBars(history.Select(x => x as Tick), requestedSymbol);
                     break;
+                case TickType.Quote when resolution == Resolution.Tick:
+                    AssertQuoteTickBars(history.Select(x => x as Tick), requestedSymbol);
+                    break;
                 case TickType.Quote:
-                    AssertTickQuoteBars(history.Select(t => t as Tick), requestedSymbol);
+                    AssertQuoteBars(history.Select(t => t as QuoteBar), requestedSymbol, resolution.ToTimeSpan());
                     break;
             }
         }
@@ -74,7 +77,7 @@ public static void AssertTradeTickBars(IEnumerable<Tick> ticks, Symbol symbol =
             }
         }
 
-        public static void AssertTickQuoteBars(IEnumerable<Tick> ticks, Symbol symbol = null)
+        public static void AssertQuoteTickBars(IEnumerable<Tick> ticks, Symbol symbol = null)
         {
             foreach (var tick in ticks)
             {
@@ -94,6 +97,41 @@ public static void AssertTickQuoteBars(IEnumerable<Tick> ticks, Symbol symbol =
             }
         }
 
+        public static void AssertQuoteBars(IEnumerable<QuoteBar> ticks, Symbol symbol, TimeSpan resolutionInTimeSpan)
+        {
+            foreach (var tick in ticks)
+            {
+                if (symbol != null)
+                {
+                    Assert.That(tick.Symbol, Is.EqualTo(symbol));
+                }
+
+                Assert.That(tick.Ask.Open, Is.GreaterThan(0));
+                Assert.That(tick.Ask.High, Is.GreaterThan(0));
+                Assert.That(tick.Ask.Low, Is.GreaterThan(0));
+                Assert.That(tick.Ask.Close, Is.GreaterThan(0));
+
+                Assert.That(tick.Bid.Open, Is.GreaterThan(0));
+                Assert.That(tick.Bid.High, Is.GreaterThan(0));
+                Assert.That(tick.Bid.Low, Is.GreaterThan(0));
+                Assert.That(tick.Bid.Close, Is.GreaterThan(0));
+
+                Assert.That(tick.Close, Is.GreaterThan(0));
+                Assert.That(tick.High, Is.GreaterThan(0));
+                Assert.That(tick.Low, Is.GreaterThan(0));
+                Assert.That(tick.Open, Is.GreaterThan(0));
+                Assert.That(tick.Price, Is.GreaterThan(0));
+                Assert.That(tick.Value, Is.GreaterThan(0));
+                Assert.That(tick.DataType, Is.EqualTo(MarketDataType.QuoteBar));
+                Assert.That(tick.EndTime, Is.GreaterThan(default(DateTime)));
+                Assert.That(tick.Time, Is.GreaterThan(default(DateTime)));
+
+                Assert.That(tick.LastAskSize, Is.GreaterThan(0));
+                Assert.That(tick.LastBidSize, Is.GreaterThan(0));
+                Assert.That(tick.Period, Is.EqualTo(resolutionInTimeSpan));
+            }
+        }
+
         public static void AssertTradeBars(IEnumerable<TradeBar> tradeBars, Symbol symbol, TimeSpan period)
         {
             foreach (var tradeBar in tradeBars)

QuantConnect.ThetaData.Tests/ThetaDataDownloaderTests.cs

@@ -23,6 +23,7 @@ namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
 
     [TestFixture]
+    [Explicit("This test requires the ThetaData terminal to be running in order to execute properly.")]
     public class ThetaDataDownloaderTests
     {
         private ThetaDataDownloader _dataDownloader;

QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs

@@ -20,6 +20,7 @@
 namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
     [TestFixture]
+    [Explicit("This test requires the ThetaData terminal to be running in order to execute properly.")]
     public class ThetaDataHistoryProviderTests
     {
         ThetaDataProvider _thetaDataProvider = new();

QuantConnect.ThetaData.Tests/ThetaDataOptionChainProviderTests.cs

@@ -25,6 +25,7 @@
 namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
     [TestFixture]
+    [Explicit("This test requires the ThetaData terminal to be running in order to execute properly.")]
     public class ThetaDataOptionChainProviderTests
     {
         private ThetaDataOptionChainProvider _thetaDataOptionChainProvider;

QuantConnect.ThetaData.Tests/ThetaDataProviderTests.cs

@@ -29,6 +29,7 @@
 namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
     [TestFixture]
+    [Explicit("This test requires the ThetaData terminal to be running in order to execute properly.")]
     public class ThetaDataProviderTests
     {
         private ThetaDataProvider _thetaDataProvider;

QuantConnect.ThetaData.Tests/ThetaDataQueueUniverseProviderTests.cs

@@ -21,6 +21,7 @@
 namespace QuantConnect.Lean.DataSource.ThetaData.Tests
 {
     [TestFixture]
+    [Explicit("This test requires the ThetaData terminal to be running in order to execute properly.")]
     public class ThetaDataQueueUniverseProviderTests
     {
         private TestableThetaDataProvider _thetaDataProvider;

QuantConnect.ThetaData.Tests/config.json

@@ -1,6 +1,4 @@
 {
   "data-folder": "../../../../Lean/Data/",
-  "data-directory": "../../../../Lean/Data/",
-
-  "thetadata-subscription-plan": "Free"
+  "thetadata-subscription-plan": "Pro"
 }

QuantConnect.ThetaData/Converters/ThetaDataQuoteListContractConverter.cs

@@ -0,0 +1,71 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using QuantConnect.Lean.DataSource.ThetaData.Models.Rest;
+
+namespace QuantConnect.Lean.DataSource.ThetaData.Converters;
+
+/// <summary>
+/// JSON converter to convert ThetaData Quote 
+/// </summary>
+public class ThetaDataQuoteListContractConverter : JsonConverter<QuoteListContract>
+{
+    /// <summary>
+    /// Gets a value indicating whether this <see cref="JsonConverter"/> can write JSON.
+    /// </summary>
+    /// <value><c>true</c> if this <see cref="JsonConverter"/> can write JSON; otherwise, <c>false</c>.</value>
+    public override bool CanWrite => false;
+
+    /// <summary>
+    /// Gets a value indicating whether this <see cref="JsonConverter"/> can read JSON.
+    /// </summary>
+    /// <value><c>true</c> if this <see cref="JsonConverter"/> can read JSON; otherwise, <c>false</c>.</value>
+    public override bool CanRead => true;
+
+    /// <summary>
+    /// Writes the JSON representation of the object.
+    /// </summary>
+    /// <param name="writer">The <see cref="JsonWriter"/> to write to.</param>
+    /// <param name="value">The value.</param>
+    /// <param name="serializer">The calling serializer.</param>
+    public override void WriteJson(JsonWriter writer, QuoteListContract value, JsonSerializer serializer)
+    {
+        throw new NotSupportedException();
+    }
+
+    /// <summary>
+    /// Reads the JSON representation of the object.
+    /// </summary>
+    /// <param name="reader">The <see cref="JsonReader"/> to read from.</param>
+    /// <param name="objectType">Type of the object.</param>
+    /// <param name="existingValue">The existing value of object being read.</param>
+    /// <param name="hasExistingValue">The existing value has a value.</param>
+    /// <param name="serializer">The calling serializer.</param>
+    /// <returns>The object value.</returns>
+    public override QuoteListContract ReadJson(JsonReader reader, Type objectType, QuoteListContract existingValue, bool hasExistingValue, JsonSerializer serializer)
+    {
+        var token = JToken.Load(reader);
+        if (token.Type != JTokenType.Array || token.Count() != 4) throw new Exception($"{nameof(ThetaDataQuoteConverter)}.{nameof(ReadJson)}: Invalid token type or count. Expected a JSON array with exactly four elements.");
+
+        return new(
+            token[0]!.Value<string>()!, 
+            token[1]!.Value<string>()!.ConvertFromThetaDataDateFormat(),
+            token[2]!.Value<decimal>(),
+            token[3]!.Value<string>()!
+            );
+    }
+}

QuantConnect.ThetaData/Models/Rest/QuoteListContract.cs

@@ -0,0 +1,39 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Newtonsoft.Json;
+using QuantConnect.Lean.DataSource.ThetaData.Converters;
+
+namespace QuantConnect.Lean.DataSource.ThetaData.Models.Rest;
+
+[JsonConverter(typeof(ThetaDataQuoteListContractConverter))]
+public readonly struct QuoteListContract
+{
+    public string Ticker { get; }
+
+    public DateTime Expiry { get; }
+
+    public decimal Strike { get; }
+
+    public string Right { get; }
+
+    public QuoteListContract(string ticker, DateTime expiry, decimal strike, string right)
+    {
+        Ticker = ticker;
+        Expiry = expiry;
+        Strike = strike;
+        Right = right;
+    }
+}

QuantConnect.ThetaData/ThetaDataDownloader.cs

@@ -129,10 +129,12 @@ public ThetaDataDownloader()
 
         protected virtual IEnumerable<Symbol> GetOptions(Symbol symbol, DateTime startUtc, DateTime endUtc)
         {
-            foreach (var option in _historyProvider.GetOptionChain(symbol, startUtc, endUtc))
-            {
-                yield return option;
-            }
+            var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+
+            return Time.EachTradeableDay(exchangeHours, startUtc.Date, endUtc.Date)
+                .Select(date => _historyProvider.GetOptionChain(symbol, date))
+                .SelectMany(x => x)
+                .Distinct();
         }
 
         /// <summary>

QuantConnect.ThetaData/ThetaDataHistoryProvider.cs

@@ -182,7 +182,12 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
                         return GetOptionEndOfDay(optionRequest,
                             // If Ask/Bid - prices/sizes zero, low quote activity, empty result, low volatility.
                             (eof) => eof.AskPrice == 0 || eof.AskSize == 0 || eof.BidPrice == 0 || eof.BidSize == 0,
-                            (quoteDateTime, eof) => new Tick(quoteDateTime, symbol, eof.AskCondition, ThetaDataExtensions.Exchanges[eof.AskExchange], eof.BidSize, eof.BidPrice, eof.AskSize, eof.AskPrice));
+                            (quoteDateTime, eof) =>
+                            {
+                                var bar = new QuoteBar(quoteDateTime, symbol, null, decimal.Zero, null, decimal.Zero, resolution.ToTimeSpan());
+                                bar.UpdateQuote(eof.BidPrice, eof.BidSize, eof.AskPrice, eof.AskSize);
+                                return bar;
+                            });
                     case TickType.OpenInterest:
                         optionRequest.Resource = "/hist/option/open_interest";
                         return GetHistoricalOpenInterestData(optionRequest, symbol);
@@ -205,7 +210,18 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
                     case TickType.Quote:
                         optionRequest.AddQueryParameter("ivl", GetIntervalsInMilliseconds(resolution));
                         optionRequest.Resource = "/hist/option/quote";
-                        return GetHistoricalQuoteData(optionRequest, symbol);
+
+                        Func<QuoteResponse, BaseData> quoteCallback = resolution == Resolution.Tick ?
+                            (quote) => new Tick(quote.DateTimeMilliseconds, symbol, quote.AskCondition, ThetaDataExtensions.Exchanges[quote.AskExchange], quote.BidSize, quote.BidPrice, quote.AskSize, quote.AskPrice)
+                            :
+                            (quote) =>
+                            {
+                                var bar = new QuoteBar(quote.DateTimeMilliseconds, symbol, null, decimal.Zero, null, decimal.Zero, resolution.ToTimeSpan());
+                                bar.UpdateQuote(quote.BidPrice, quote.BidSize, quote.AskPrice, quote.AskSize);
+                                return bar;
+                            };
+
+                        return GetHistoricalQuoteData(optionRequest, symbol, quoteCallback);
                     default:
                         throw new ArgumentException($"Invalid tick type: {tickType}.");
                 }
@@ -234,7 +250,7 @@ private IEnumerable<Tick> GetHistoricalTickTradeData(RestRequest request, Symbol
             }
         }
 
-        private IEnumerable<BaseData> GetHistoricalQuoteData(RestRequest request, Symbol symbol)
+        private IEnumerable<BaseData> GetHistoricalQuoteData(RestRequest request, Symbol symbol, Func<QuoteResponse, BaseData> callback)
         {
             foreach (var quotes in _restApiClient.ExecuteRequest<BaseResponse<QuoteResponse>>(request))
             {
@@ -246,7 +262,7 @@ private IEnumerable<BaseData> GetHistoricalQuoteData(RestRequest request, Symbol
                         continue;
                     }
 
-                    yield return new Tick(quote.DateTimeMilliseconds, symbol, quote.AskCondition, ThetaDataExtensions.Exchanges[quote.AskExchange], quote.BidSize, quote.BidPrice, quote.AskSize, quote.AskPrice);
+                    yield return callback(quote);
                 }
             }
         }

QuantConnect.ThetaData/ThetaDataOptionChainProvider.cs

@@ -16,6 +16,7 @@
 using RestSharp;
 using QuantConnect.Logging;
 using QuantConnect.Interfaces;
+using QuantConnect.Lean.DataSource.ThetaData.Models.Rest;
 using QuantConnect.Lean.DataSource.ThetaData.Models.Common;
 
 namespace QuantConnect.Lean.DataSource.ThetaData
@@ -75,43 +76,16 @@ public IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
             var optionsSecurityType = underlying.SecurityType == SecurityType.Index ? SecurityType.IndexOption : SecurityType.Option;
             var optionStyle = optionsSecurityType.DefaultOptionStyle();
 
-            var strikeRequest = new RestRequest("/list/strikes", Method.GET);
-            strikeRequest.AddQueryParameter("root", underlying.Value);
-            foreach (var expiryDateStr in GetExpirationDates(underlying.Value))
-            {
-                var expirationDate = expiryDateStr.ConvertFromThetaDataDateFormat();
-
-                if (expirationDate < date)
-                {
-                    continue;
-                }
+            // just using quote, which is the most inclusive
+            var request = new RestRequest($"/list/contracts/option/quote", Method.GET);
 
-                strikeRequest.AddOrUpdateParameter("exp", expirationDate.ConvertToThetaDataDateFormat());
-
-                foreach (var strike in _restApiClient.ExecuteRequest<BaseResponse<decimal>>(strikeRequest).SelectMany(strikes => strikes.Response))
-                {
-                    foreach (var right in optionRights)
-                    {
-                        yield return _symbolMapper.GetLeanSymbol(underlying.Value, optionsSecurityType, underlying.ID.Market, optionStyle,
-                            expirationDate, strike, right, underlying);
-                    }
-                }
-            }
-        }
-
-        /// <summary>
-        /// Returns all expirations date for a ticker.
-        /// </summary>
-        /// <param name="ticker">The underlying symbol value to list expirations for.</param>
-        /// <returns>An enumerable collection of expiration dates in string format (e.g., "20240303" for March 3, 2024).</returns>
-        public IEnumerable<string> GetExpirationDates(string ticker)
-        {
-            var request = new RestRequest("/list/expirations", Method.GET);
-            request.AddQueryParameter("root", ticker);
+            request.AddQueryParameter("start_date", date.ConvertToThetaDataDateFormat());
+            request.AddQueryParameter("root", underlying.Value);
 
-            foreach (var expirationDate in _restApiClient.ExecuteRequest<BaseResponse<string>>(request).SelectMany(x => x.Response))
+            foreach (var option in _restApiClient.ExecuteRequest<BaseResponse<QuoteListContract>>(request).SelectMany(x => x.Response))
             {
-                yield return expirationDate;
+                yield return _symbolMapper.GetLeanSymbol(underlying.Value, optionsSecurityType, underlying.ID.Market, optionStyle,
+                    option.Expiry, option.Strike, option.Right == "C" ? OptionRight.Call : OptionRight.Put, underlying);
             }
         }
     }