Skip to content

Commit

Permalink
Merge pull request #96 from jhonabreul/feature-null-history-for-unsup…
Browse files Browse the repository at this point in the history 
…ported-securities

Return null on unsupported history requests
  • Loading branch information
@jhonabreul
jhonabreul authored Feb 27, 2024
2 parents f7f2fd3 + e3e5b52 commit fdbe0ca
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 52 deletions.
56 changes: 54 additions & 2 deletions QuantConnect.InteractiveBrokersBrokerage.Tests/InteractiveBrokersBrokerageAdditionalTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,14 +738,66 @@ public void GetHistoryDoesNotThrowError504WhenDisconnected()
DataNormalizationMode.Raw,
TickType.Trade);

var history = brokerage.GetHistory(request).ToList();
var history = brokerage.GetHistory(request);

Assert.AreEqual(0, history.Count);
Assert.IsNull(history);

Assert.IsFalse(hasError);
}
}

private static TestCaseData[] UnsupportedHistoryTestCases => new[]
{
// Canonicals are not supported
new TestCaseData(Symbol.CreateCanonicalOption(Symbols.AAPL), Resolution.Daily, TickType.Trade),
new TestCaseData(Symbols.CreateFuturesCanonicalSymbol("ES"), Resolution.Daily, TickType.Trade),
// Unsupported markets
new TestCaseData(Symbol.Create("SPY", SecurityType.Equity, Market.India), Resolution.Daily, TickType.Trade),
new TestCaseData(Symbol.Create("EURUSD", SecurityType.Forex, Market.Binance), Resolution.Daily, TickType.Trade),
new TestCaseData(Symbol.CreateOption(Symbols.SPY, Market.India, OptionStyle.American, OptionRight.Call, 100m, new DateTime(2024, 12, 12)), Resolution.Daily, TickType.Trade),
new TestCaseData(Symbol.CreateOption(Symbols.SPX, Market.India, OptionStyle.American, OptionRight.Call, 100m, new DateTime(2024, 12, 12)), Resolution.Daily, TickType.Trade),
new TestCaseData(Symbol.Create("SPX", SecurityType.Index, Market.India), Resolution.Daily, TickType.Trade),
// Unsupported resolution
new TestCaseData(Symbols.SPY, Resolution.Tick, TickType.Trade),
new TestCaseData(Symbols.SPY_C_192_Feb19_2016, Resolution.Tick, TickType.Trade),
new TestCaseData(Symbols.USDJPY, Resolution.Tick, TickType.Trade),
new TestCaseData(Symbols.SPX, Resolution.Tick, TickType.Trade),
new TestCaseData(Symbols.Future_ESZ18_Dec2018, Resolution.Tick, TickType.Trade),
// Unsupported tick type
new TestCaseData(Symbols.SPY, Resolution.Tick, TickType.OpenInterest),
new TestCaseData(Symbols.SPY_C_192_Feb19_2016, Resolution.Tick, TickType.OpenInterest),
new TestCaseData(Symbols.USDJPY, Resolution.Tick, TickType.OpenInterest),
new TestCaseData(Symbols.SPX, Resolution.Tick, TickType.OpenInterest),
new TestCaseData(Symbols.Future_ESZ18_Dec2018, Resolution.Tick, TickType.OpenInterest),
};

[TestCaseSource(nameof(UnsupportedHistoryTestCases))]
public void GetHistoryReturnsNullForUnsupportedCases(Symbol symbol, Resolution resolution, TickType tickType)
{
using (var brokerage = GetBrokerage())
{
Assert.IsTrue(brokerage.IsConnected);

var request = new HistoryRequest(
new DateTime(2021, 1, 1).ConvertToUtc(TimeZones.NewYork),
new DateTime(2021, 1, 27).ConvertToUtc(TimeZones.NewYork),
typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
tickType);

var history = brokerage.GetHistory(request);

Assert.IsNull(history);
}
}

[TestCase("0.00:01:01.000", Resolution.Tick, "100 S")]
[TestCase("1.00:00:00.000", Resolution.Tick, "2000 S")]
[TestCase("1.00:01:00.000", Resolution.Tick, "2000 S")]
Expand Down
56 changes: 29 additions & 27 deletions QuantConnect.InteractiveBrokersBrokerage.ToolBox/IBDataDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,14 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

if (tickType != TickType.Quote)
if (tickType != TickType.Quote || resolution == Resolution.Tick)
{
yield break;
}

if (resolution == Resolution.Tick)
{
throw new NotSupportedException("Resolution not available: " + resolution);
return null;
}

if (endUtc < startUtc)
{
throw new ArgumentException("The end date must be greater or equal than the start date.");
return Enumerable.Empty<BaseData>();
}

var symbols = new List<Symbol>{ symbol };
Expand All @@ -84,26 +79,33 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = MarketHoursDatabase.FromDataFolder().GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);

foreach (var targetSymbol in symbols)
{
var historyRequest = new HistoryRequest(startUtc,
endUtc,
typeof(QuoteBar),
targetSymbol,
resolution,
exchangeHours: exchangeHours,
dataTimeZone: dataTimeZone,
resolution,
includeExtendedMarketHours: true,
false,
DataNormalizationMode.Adjusted,
TickType.Quote);

foreach (var baseData in _brokerage.GetHistory(historyRequest))
return symbols
.Select(symbol =>
{
yield return baseData;
}
}
var request = new HistoryRequest(startUtc,
endUtc,
typeof(QuoteBar),
symbol,
resolution,
exchangeHours: exchangeHours,
dataTimeZone: dataTimeZone,
resolution,
includeExtendedMarketHours: true,
false,
DataNormalizationMode.Adjusted,
TickType.Quote);

var history = _brokerage.GetHistory(request);

if (history == null)
{
Logging.Log.Trace($"IBDataDownloader.Get(): Ignoring history request for unsupported symbol {symbol}");
}

return history;
})
.Where(history => history != null)
.SelectMany(history => history);
}

/// <summary>
Expand Down
4 changes: 4 additions & 0 deletions QuantConnect.InteractiveBrokersBrokerage.ToolBox/IBDownloaderProgram.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ public static void IBDownloader(IList<string> tickers, string resolution, DateTi
while (startDate < auxEndDate)
{
var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, startDate, auxEndDate, TickType.Quote));
if (data == null)
{
break;
}
var bars = data.Cast<QuoteBar>().ToList();

if (allResolutions)
Expand Down
53 changes: 30 additions & 23 deletions QuantConnect.InteractiveBrokersBrokerage/InteractiveBrokersBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ public sealed class InteractiveBrokersBrokerage : Brokerage, IDataQueueHandler,
private bool _historyDelistedAssetWarning;
private bool _historyExpiredAssetWarning;
private bool _historyOpenInterestWarning;
private bool _historyInvalidPeriodWarning;

/// <summary>
/// Returns true if we're currently connected to the broker
Expand Down Expand Up @@ -4050,27 +4051,13 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
BrokerageMessageType.Warning,
"GetHistoryWhenDisconnected",
"History requests cannot be submitted when disconnected."));
yield break;
return null;
}

// skipping universe and canonical symbols
if (!CanSubscribe(request.Symbol) ||
(request.Symbol.ID.SecurityType.IsOption() && request.Symbol.IsCanonical()))
if (!CanSubscribe(request.Symbol))
{
yield break;
}

// skip invalid security types
if (request.Symbol.SecurityType != SecurityType.Equity &&
request.Symbol.SecurityType != SecurityType.Index &&
request.Symbol.SecurityType != SecurityType.Forex &&
request.Symbol.SecurityType != SecurityType.Cfd &&
request.Symbol.SecurityType != SecurityType.Future &&
request.Symbol.SecurityType != SecurityType.FutureOption &&
request.Symbol.SecurityType != SecurityType.Option &&
request.Symbol.SecurityType != SecurityType.IndexOption)
{
yield break;
return null;
}

// tick resolution not supported for now
Expand All @@ -4079,18 +4066,28 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
// TODO: upgrade IB C# API DLL
// In IB API version 973.04, the reqHistoricalTicks function has been added,
// which would now enable us to support history requests at Tick resolution.
yield break;
return null;
}


if (request.TickType == TickType.OpenInterest)
{
if (!_historyOpenInterestWarning)
{
_historyOpenInterestWarning = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "GetHistoryOpenInterest", "IB does not provide open interest historical data"));
}
yield break;
return null;
}

if (request.EndTimeUtc < request.StartTimeUtc)
{
if (!_historyInvalidPeriodWarning)
{
_historyInvalidPeriodWarning = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning,
"GetHistoryEndTimeBeforeStartTime", "The history request end time is before the start time. No history will be returned."));
}
return null;
}

// See https://interactivebrokers.github.io/tws-api/historical_limitations.html
Expand Down Expand Up @@ -4118,7 +4115,7 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
_historyExpiredAssetWarning = true;
}
Log.Trace($"InteractiveBrokersBrokerage::GetHistory(): Skip request of expired asset: {request.Symbol.Value}. {localNow} > {historicalLimitDate}");
yield break;
return Enumerable.Empty<BaseData>();
}
}
else if(request.Symbol.ID.SecurityType == SecurityType.Equity)
Expand All @@ -4135,7 +4132,7 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
}

Log.Trace($"InteractiveBrokersBrokerage::GetHistory(): Skip request of delisted asset: {request.Symbol.Value}. DelistingDate: {mapfile.DelistingDate}");
yield break;
return Enumerable.Empty<BaseData>();
}
}

Expand Down Expand Up @@ -4174,7 +4171,7 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
if (startTime > endTime)
{
Log.Trace($"InteractiveBrokersBrokerage::GetHistory(): Skip too old 'Resolution.Second' request {startTimeLocal} < {minimumLocalStartTime}");
yield break;
return Enumerable.Empty<BaseData>();
}
}
}
Expand Down Expand Up @@ -4207,6 +4204,16 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
history = GetHistory(request, contract, startTime, endTime, exchangeTimeZone, resolution, HistoricalDataType.Trades);
}

return FilterHistory(history, request, startTimeLocal, endTimeLocal, contract);
}

private IEnumerable<BaseData> FilterHistory(
IEnumerable<BaseData> history,
HistoryRequest request,
DateTime startTimeLocal,
DateTime endTimeLocal,
Contract contract)
{
// cleaning the data before returning it back to user
foreach (var bar in history.Where(bar => bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal))
{
Expand Down

0 comments on commit fdbe0ca

Please sign in to comment.