diff --git a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs index ee28f06..ec5f61f 100644 --- a/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs +++ b/QuantConnect.IEX.Tests/IEXDataHistoryTests.cs @@ -14,6 +14,7 @@ */ using System; +using NodaTime; using System.Linq; using NUnit.Framework; using QuantConnect.Data; @@ -65,7 +66,7 @@ internal static IEnumerable ValidDataTestParameters .SetDescription("Valid parameters - Minute resolution, 5 days period.") .SetCategory("Valid"); - yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45), true) + yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45)) .SetDescription("Valid parameters - Beyond 45 days, Minute resolution.") .SetCategory("Valid"); } @@ -159,6 +160,22 @@ public void IEXCloudGetHistoryWithValidDataTestParameters(Symbol symbol, Resolut Assert.That(slices, Is.Ordered.By("Time")); } + [Explicit("This tests require a iexcloud.io api key")] + [TestCase("GOOGL", Resolution.Daily, "2013/1/3", "2015/12/29", Description = "October 2, 2015. [GOOG -> GOOGL]")] + [TestCase("META", Resolution.Daily, "2020/1/3", "2023/12/29", Description = "October 28, 2021. [FB -> META]")] + public void GetAncientEquityHistoricalData(string ticker, Resolution resolution, DateTime startDate, DateTime endDate) + { + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + + var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDate, endDate); + + var slices = iexDataProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList(); + + Assert.Greater(slices.Count, 1); + Assert.That(slices.First().Time.Date, Is.EqualTo(startDate)); + Assert.That(slices.Last().Time.Date, Is.LessThanOrEqualTo(endDate)); + } + internal static void AssertTradeBar(Symbol expectedSymbol, Resolution resolution, BaseData baseData, Symbol actualSymbol = null) { if (actualSymbol != null) @@ -235,25 +252,42 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period) { - var utcNow = DateTime.UtcNow; + var end = new DateTime(2024, 3, 15, 16, 0, 0); - var dataType = LeanData.GetDataType(resolution, tickType); + if (resolution == Resolution.Daily) + { + end = end.Date.AddDays(1); + } + + return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end); + } + + internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime, + SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null) + { + if (exchangeHours == null) + { + exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork); + } - var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); - var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + if (dataTimeZone == null) + { + dataTimeZone = TimeZones.NewYork; + } + var dataType = LeanData.GetDataType(resolution, tickType); return new HistoryRequest( - startTimeUtc: utcNow.Add(-period), - endTimeUtc: utcNow, + startTimeUtc: startDateTime, + endTimeUtc: endDateTime, dataType: dataType, symbol: symbol, resolution: resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, - fillForwardResolution: resolution, + fillForwardResolution: null, includeExtendedMarketHours: true, isCustomData: false, - DataNormalizationMode.Raw, + dataNormalizationMode: DataNormalizationMode.Adjusted, tickType: tickType ); } diff --git a/QuantConnect.IEX/IEXDataDownloader.cs b/QuantConnect.IEX/IEXDataDownloader.cs index 826165b..f1905b9 100644 --- a/QuantConnect.IEX/IEXDataDownloader.cs +++ b/QuantConnect.IEX/IEXDataDownloader.cs @@ -14,8 +14,8 @@ */ using QuantConnect.Data; +using QuantConnect.Util; using QuantConnect.Securities; -using QuantConnect.Data.Market; namespace QuantConnect.Lean.DataSource.IEX { @@ -44,30 +44,25 @@ public void Dispose() public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters) { var symbol = dataDownloaderGetParameters.Symbol; - var resolution = dataDownloaderGetParameters.Resolution; - var startUtc = dataDownloaderGetParameters.StartUtc; - var endUtc = dataDownloaderGetParameters.EndUtc; - var tickType = dataDownloaderGetParameters.TickType; var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - var historyRequests = new[] { - new HistoryRequest(startUtc, - endUtc, - typeof(TradeBar), - symbol, - resolution, - exchangeHours, - dataTimeZone, - resolution, - true, - false, - DataNormalizationMode.Raw, - TickType.Trade) - }; + var historyRequests = new HistoryRequest( + dataDownloaderGetParameters.StartUtc, + dataDownloaderGetParameters.EndUtc, + LeanData.GetDataType(dataDownloaderGetParameters.Resolution, dataDownloaderGetParameters.TickType), + symbol, + dataDownloaderGetParameters.Resolution, + exchangeHours, + dataTimeZone, + dataDownloaderGetParameters.Resolution, + true, + false, + DataNormalizationMode.Raw, + dataDownloaderGetParameters.TickType); - return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]); + return _handler.ProcessHistoryRequests(historyRequests); } } } diff --git a/QuantConnect.IEX/IEXDataProvider.cs b/QuantConnect.IEX/IEXDataProvider.cs index 21e1a70..c776ee6 100644 --- a/QuantConnect.IEX/IEXDataProvider.cs +++ b/QuantConnect.IEX/IEXDataProvider.cs @@ -56,22 +56,28 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler /// /// Flag indicating whether a warning about unsupported data types in user history should be suppressed to prevent spam. /// - private static bool _invalidHistoryDataTypeWarningFired; + private volatile bool _invalidHistoryDataTypeWarningFired; /// /// Indicates whether the warning for invalid has been fired. /// - private bool _invalidSecurityTypeWarningFired; + private volatile bool _invalidSecurityTypeWarningFired; /// /// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - private bool _invalidStartTimeWarningFired; + private volatile bool _invalidStartTimeWarningFired; /// /// Indicates whether a warning for an invalid has been fired, where the resolution is neither daily nor minute-based. /// - private bool _invalidResolutionWarningFired; + private volatile bool _invalidResolutionWarningFired; + + /// + /// Indicates whether a warning has been triggered for reaching the limit of resolution, + /// where the startDateTime is not greater than 2 years. + /// + private volatile bool _limitMinuteResolutionWarningFired; /// /// Represents two clients: one for the trade channel and another for the top-of-book channel. @@ -512,49 +518,13 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List(); foreach (var request in requests) { - // IEX does return historical TradeBar - give one time warning if inconsistent data type was requested - if (request.TickType != TickType.Trade) - { - if (!_invalidHistoryDataTypeWarningFired) - { - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " + - "Currently available support only for historical of type - TradeBar"); - _invalidHistoryDataTypeWarningFired = true; - } - continue; - } - - if (!CanSubscribe(request.Symbol)) - { - if (!_invalidSecurityTypeWarningFired) - { - Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); - _invalidSecurityTypeWarningFired = true; - } - continue; - } - - if (request.StartTimeUtc >= request.EndTimeUtc) - { - if (!_invalidStartTimeWarningFired) - { - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); - _invalidStartTimeWarningFired = true; - } - continue; - } + var history = ProcessHistoryRequests(request); - if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) + if (history == null) { - if (!_invalidResolutionWarningFired) - { - Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); - _invalidResolutionWarningFired = true; - } continue; } - var history = ProcessHistoryRequests(request); var subscription = CreateSubscription(request, history); subscriptions.Add(subscription); } @@ -570,23 +540,75 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// /// Populate request data /// - private IEnumerable? ProcessHistoryRequests(Data.HistoryRequest request) + public IEnumerable? ProcessHistoryRequests(Data.HistoryRequest request) { - var ticker = request.Symbol.ID.Symbol; - var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); - var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc); + // IEX does return historical TradeBar - give one time warning if inconsistent data type was requested + if (request.TickType != TickType.Trade) + { + if (!_invalidHistoryDataTypeWarningFired) + { + _invalidHistoryDataTypeWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " + + "Currently available support only for historical of type - TradeBar"); + } + return null; + } + + if (!CanSubscribe(request.Symbol)) + { + if (!_invalidSecurityTypeWarningFired) + { + _invalidSecurityTypeWarningFired = true; + Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'"); + } + return null; + } - Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}]."); + if (request.StartTimeUtc >= request.EndTimeUtc) + { + if (!_invalidStartTimeWarningFired) + { + _invalidStartTimeWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned."); + } + return null; + } + + if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) + { + if (!_invalidResolutionWarningFired) + { + _invalidResolutionWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution."); + } + return null; + } + + // Always obtain the most relevant ticker symbol based on the current time. + var ticker = SecurityIdentifier.Ticker(request.Symbol, DateTime.UtcNow); + var startExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc); + var endExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc); + + if (request.Resolution == Resolution.Minute && startExchangeDateTime <= DateTime.Today.AddYears(-2)) + { + if (!_limitMinuteResolutionWarningFired) + { + _limitMinuteResolutionWarningFired = true; + Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls with minute resolution for IEX available only not more 2 years."); + } + return null; + } + + Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{startExchangeDateTime} - {endExchangeDateTime}]."); - var span = end - start; var urls = new List(); switch (request.Resolution) { case Resolution.Minute: { - var begin = start; - while (begin < end) + var begin = startExchangeDateTime; + while (begin < endExchangeDateTime) { var url = $"{BaseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}"; urls.Add(url); @@ -597,35 +619,19 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } case Resolution.Daily: { - string suffix; - if (span.Days < 30) - { - suffix = "1m"; - } - else if (span.Days < 3 * 30) - { - suffix = "3m"; - } - else if (span.Days < 6 * 30) - { - suffix = "6m"; - } - else if (span.Days < 12 * 30) - { - suffix = "1y"; - } - else if (span.Days < 24 * 30) - { - suffix = "2y"; - } - else if (span.Days < 60 * 30) - { - suffix = "5y"; - } - else + // To retrieve a specific start-to-end dateTime range, calculate the duration between the current time and the startExchangeDateTime. + var span = DateTime.Now - startExchangeDateTime; + + string suffix = span.Days switch { - suffix = "max"; // max is 15 years - } + < 30 => "1m", + < 3 * 30 => "3m", + < 6 * 30 => "6m", + < 12 * 30 => "1y", + < 24 * 30 => "2y", + < 60 * 30 => "5y", + _ => "max" // max is 15 years + }; var url = $"{BaseUrl}/{ticker}/chart/{suffix}?token={_apiKey}"; urls.Add(url); @@ -634,7 +640,9 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } } - return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol); + Log.Debug($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {string.Join("\n", urls)}"); + + return GetHistoryRequestByUrls(urls, startExchangeDateTime, endExchangeDateTime, request.Resolution, request.DataNormalizationMode, request.Symbol); } /// @@ -678,7 +686,7 @@ private IEnumerable GetHistoryRequestByUrls(List urls, DateTim period = TimeSpan.FromDays(1); } - if (date < startDateTime || date > endDateTime) + if (!(date >= startDateTime && date < endDateTime)) { continue; } @@ -710,9 +718,7 @@ private IEnumerable GetHistoryRequestByUrls(List urls, DateTim volume = item["volume"].Value(); } - var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period); - - yield return tradeBar; + yield return new TradeBar(ConvertTickTimeBySymbol(symbol, date), symbol, open, high, low, close, volume, period); } } }