Skip to content

Commit

Permalink
Ignore expired futures history request error from IB (#89)
Browse files Browse the repository at this point in the history 
* Ignore expired futures history request error from IB

* Add unit tests

* Minor changes
  • Loading branch information
@jhonabreul
jhonabreul authored Oct 31, 2023
1 parent a405fb0 commit d8a6db8
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 15 deletions.
40 changes: 40 additions & 0 deletions QuantConnect.InteractiveBrokersBrokerage.Tests/InteractiveBrokersBrokerageAdditionalTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using NodaTime;
using NUnit.Framework;
using QuantConnect.Algorithm;
using QuantConnect.Brokerages;
using QuantConnect.Brokerages.InteractiveBrokers;
using QuantConnect.Data;
using QuantConnect.Data.Market;
Expand Down Expand Up @@ -676,6 +677,45 @@ public void GetHistoryData(
Assert.AreEqual(expectedCount, result.Count);
}

[Test]
public void IgnoresSecurityNotFoundErrorOnExpiredContractsHistoricalRequests()
{
using var brokerage = GetBrokerage();
Assert.IsTrue(brokerage.IsConnected);

var messages = new List<BrokerageMessageEvent>();
void onMessage(object sender, BrokerageMessageEvent e)
{
messages.Add(e);
}

brokerage.Message += onMessage;

var request = new HistoryRequest(
new DateTime(2023, 09, 04, 9, 30, 0).ConvertToUtc(TimeZones.NewYork),
new DateTime(2023, 09, 14, 16, 0, 0).ConvertToUtc(TimeZones.NewYork),
typeof(TradeBar),
Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new DateTime(2023, 09, 15)),
Resolution.Minute,
SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
TimeZones.NewYork,
null,
false,
false,
DataNormalizationMode.Raw,
TickType.Trade);

var history = brokerage.GetHistory(request).ToList();

Assert.AreEqual(0, history.Count);

Assert.IsFalse(messages.Any(x => x.Type == BrokerageMessageType.Error), string.Join("\n", messages.Select(x => x.Message)));

Console.WriteLine(string.Join("\n", messages.Select(x => x.Message)));

brokerage.Message -= onMessage;
}

private List<BaseData> GetHistory(
Symbol symbol,
Resolution resolution,
Expand Down
112 changes: 100 additions & 12 deletions QuantConnect.InteractiveBrokersBrokerage/InteractiveBrokersBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ public sealed class InteractiveBrokersBrokerage : Brokerage, IDataQueueHandler,
private readonly SemaphoreSlim _concurrentHistoryRequests = new (20);

// additional IB request information, will be matched with errors in the handler, for better error reporting
private readonly ConcurrentDictionary<int, string> _requestInformation = new ConcurrentDictionary<int, string>();
private readonly ConcurrentDictionary<int, RequestInformation> _requestInformation = new();

// when unsubscribing symbols immediately after subscribing IB returns an error (Can't find EId with tickerId:nnn),
// so we track subscription times to ensure symbols are not unsubscribed before a minimum time span has elapsed
Expand Down Expand Up @@ -443,7 +443,13 @@ public override bool CancelOrder(Order order)
{
var orderId = Parse.Int(id);

_requestInformation[orderId] = $"[Id={orderId}] CancelOrder: " + order;
_requestInformation[orderId] = new RequestInformation
{
RequestId = orderId,
RequestType = RequestType.CancelOrder,
AssociatedSymbol = order.Symbol,
Message = $"[Id={orderId}] CancelOrder: " + order
};

CheckRateLimiting();

Expand Down Expand Up @@ -666,7 +672,12 @@ public override List<CashAmount> GetCashBalance()

var requestId = GetNextId();

_requestInformation[requestId] = $"[Id={requestId}] GetExecutions: " + symbol;
_requestInformation[requestId] = new RequestInformation
{
RequestId = requestId,
RequestType = RequestType.Executions,
Message = $"[Id={requestId}] GetExecutions: " + symbol
};

// define our event handlers
EventHandler<IB.RequestEndEventArgs> clientOnExecutionDataEnd = (sender, args) =>
Expand Down Expand Up @@ -1301,7 +1312,13 @@ private void IBPlaceOrder(Order order, bool needsNewId, string exchange = null)
throw new ArgumentException("Expected order with populated BrokerId for updating orders.");
}

_requestInformation[ibOrderId] = $"[Id={ibOrderId}] IBPlaceOrder: {order.Symbol.Value} ({GetContractDescription(contract)} )";
_requestInformation[ibOrderId] = new RequestInformation
{
RequestId = ibOrderId,
RequestType = RequestType.PlaceOrder,
AssociatedSymbol = order.Symbol,
Message = $"[Id={ibOrderId}] IBPlaceOrder: {order.Symbol.Value} ({GetContractDescription(contract)} )"
};

CheckRateLimiting();

Expand Down Expand Up @@ -1450,7 +1467,12 @@ private ContractDetails GetContractDetailsImpl(Contract contract, string ticker)

Log.Trace($"InteractiveBrokersBrokerage.GetContractDetails(): {ticker} ({contract})");

_requestInformation[requestId] = $"[Id={requestId}] GetContractDetails: {ticker} ({contract})";
_requestInformation[requestId] = new RequestInformation
{
RequestId = requestId,
RequestType = RequestType.ContractDetails,
Message = $"[Id={requestId}] GetContractDetails: {ticker} ({contract})"
};

var manualResetEvent = new ManualResetEvent(false);

Expand Down Expand Up @@ -1553,7 +1575,12 @@ public IEnumerable<ContractDetails> FindContracts(Contract contract, string tick

var requestId = GetNextId();

_requestInformation[requestId] = $"[Id={requestId}] FindContracts: {ticker} ({GetContractDescription(contract)})";
_requestInformation[requestId] = new RequestInformation
{
RequestId = requestId,
RequestType = RequestType.FindContracts,
Message = $"[Id={requestId}] FindContracts: {ticker} ({GetContractDescription(contract)})"
}; ;

var manualResetEvent = new ManualResetEvent(false);
var contractDetails = new List<ContractDetails>();
Expand Down Expand Up @@ -1623,10 +1650,9 @@ private void HandleError(object sender, IB.ErrorEventArgs e)
errorMsg = errorMsg.Replace("\r\n", ". ").Replace("\r", ". ").Replace("\n", ". ");

// if there is additional information for the originating request, append it to the error message
string requestMessage;
if (_requestInformation.TryGetValue(requestId, out requestMessage))
if (_requestInformation.TryGetValue(requestId, out var requestInfo))
{
errorMsg += ". Origin: " + requestMessage;
errorMsg += ". Origin: " + requestInfo.Message;
}

// historical data request with no data returned
Expand Down Expand Up @@ -1715,6 +1741,28 @@ private void HandleError(object sender, IB.ErrorEventArgs e)
errorMsg = $"{e.Message} - The current number of active market data subscriptions in TWS and the API altogether has been exceeded ({_subscribedSymbols.Count}). This number is calculated based on a formula which is based on the equity, commissions, and quote booster packs in an account.";
_maxSubscribedSymbolsReached = true;
}
else if (errorCode == 200)
{
// No security definition has been found for the request
// This is a common error when requesting historical data for expired contracts, in which case can ignore it
if (requestInfo is not null && requestInfo.RequestType == RequestType.History)
{
MapFile mapFile = null;
if (requestInfo.AssociatedSymbol.RequiresMapping())
{
var resolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(requestInfo.AssociatedSymbol));
mapFile = resolver.ResolveMapFile(requestInfo.AssociatedSymbol);
}
var historicalLimitDate = requestInfo.AssociatedSymbol.GetDelistingDate(mapFile).AddDays(1)
.ConvertToUtc(requestInfo.HistoryRequest.ExchangeHours.TimeZone);

if (DateTime.UtcNow.Date > historicalLimitDate)
{
Log.Trace($"InteractiveBrokersBrokerage.HandleError(): Expired contract historical data request, ignoring error. ErrorCode: {errorCode} - {errorMsg}");
return;
}
}
}

if (InvalidatingCodes.Contains(errorCode))
{
Expand Down Expand Up @@ -3228,7 +3276,9 @@ private Symbol MapSymbol(Contract contract)
market = defaultMarket;
}

var contractExpiryDate = DateTime.ParseExact(contract.LastTradeDateOrContractMonth, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
var contractExpiryDate = !string.IsNullOrEmpty(contract.LastTradeDateOrContractMonth)
? DateTime.ParseExact(contract.LastTradeDateOrContractMonth, DateFormat.EightCharacter, CultureInfo.InvariantCulture)
: SecurityIdentifier.DefaultDate;

if (!isFutureOption)
{
Expand Down Expand Up @@ -3401,7 +3451,13 @@ private bool Subscribe(IEnumerable<Symbol> symbols)
var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(subscribeSymbol.ID.Market, subscribeSymbol, subscribeSymbol.SecurityType, Currencies.USD);
var priceMagnifier = symbolProperties.PriceMagnifier;

_requestInformation[id] = $"[Id={id}] Subscribe: {symbol.Value} ({GetContractDescription(contract)})";
_requestInformation[id] = new RequestInformation
{
RequestId = id,
RequestType = RequestType.Subscription,
AssociatedSymbol = symbol,
Message = $"[Id={id}] Subscribe: {symbol.Value} ({GetContractDescription(contract)})"
};

CheckRateLimiting();

Expand Down Expand Up @@ -4124,7 +4180,14 @@ private IEnumerable<TradeBar> GetHistory(
TradeBar oldestDataPoint = null;
var historicalTicker = GetNextId();

_requestInformation[historicalTicker] = $"[Id={historicalTicker}] GetHistory: {request.Symbol.Value} ({GetContractDescription(contract)})";
_requestInformation[historicalTicker] = new RequestInformation
{
RequestId = historicalTicker,
RequestType = RequestType.History,
AssociatedSymbol = request.Symbol,
Message = $"[Id={historicalTicker}] GetHistory: {request.Symbol.Value} ({GetContractDescription(contract)})",
HistoryRequest = request
};

EventHandler<IB.HistoricalDataEventArgs> clientOnHistoricalData = (sender, args) =>
{
Expand Down Expand Up @@ -4883,5 +4946,30 @@ private static class AccountValueKeys
private static readonly TimeSpan _defaultRestartDelay = TimeSpan.FromMinutes(5);

private static TimeSpan _defaultWeeklyRestartUtcTime = GetNextWeekendReconnectionTimeUtc().TimeOfDay;

private enum RequestType
{
PlaceOrder,
UpdateOrder,
CancelOrder,
Subscription,
FindContracts,
ContractDetails,
History,
Executions,
}

private class RequestInformation
{
public int RequestId { get; set; }

public RequestType RequestType { get; set; }

public Symbol? AssociatedSymbol { get; set; }

public string Message { get; set; }

public HistoryRequest? HistoryRequest { get; set; }
}
}
}
7 changes: 4 additions & 3 deletions QuantConnect.InteractiveBrokersBrokerage/QuantConnect.InteractiveBrokersBrokerage.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@
</ItemGroup>

<ItemGroup>
<None Include="InteractiveBrokers\IB-symbol-map.json" Pack="True">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="InteractiveBrokers\IB-symbol-map.json" Pack="True">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<PackageCopyToOutput>true</PackageCopyToOutput>
</None>
<Reference Include="CSharpAPI">
<HintPath>CSharpAPI.dll</HintPath>
</Reference>
Expand Down

0 comments on commit d8a6db8

Please sign in to comment.