diff --git a/QuantConnect.KrakenBrokerage.Tests/KrakenBrokerageDataQueueHandlerTests.cs b/QuantConnect.KrakenBrokerage.Tests/KrakenBrokerageDataQueueHandlerTests.cs index ca85976..35fa04e 100644 --- a/QuantConnect.KrakenBrokerage.Tests/KrakenBrokerageDataQueueHandlerTests.cs +++ b/QuantConnect.KrakenBrokerage.Tests/KrakenBrokerageDataQueueHandlerTests.cs @@ -29,6 +29,7 @@ private static TestCaseData[] TestParameters { get { + TestGlobals.Initialize(); return new[] { new TestCaseData(Symbol.Create("EURUSD", SecurityType.Crypto, Market.Kraken), Resolution.Tick, 10000, false), diff --git a/QuantConnect.KrakenBrokerage.Tests/KrakenHistoryProviderTests.cs b/QuantConnect.KrakenBrokerage.Tests/KrakenHistoryProviderTests.cs index 70fbf2b..cba67c5 100644 --- a/QuantConnect.KrakenBrokerage.Tests/KrakenHistoryProviderTests.cs +++ b/QuantConnect.KrakenBrokerage.Tests/KrakenHistoryProviderTests.cs @@ -20,8 +20,6 @@ using QuantConnect.Brokerages.Kraken; using QuantConnect.Data; using QuantConnect.Data.Market; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Securities; @@ -29,97 +27,58 @@ namespace QuantConnect.Tests.Brokerages.Kraken { public partial class KrakenBrokerageTests { - [Test] - [TestCaseSource(nameof(ValidHistory))] - [TestCaseSource(nameof(InvalidHistory))] - public void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException) + private static Symbol _ethusd; + private static Symbol ETHUSD { - TestDelegate test = () => + get { - var brokerage = (KrakenBrokerage)Brokerage; - - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), null)); - - var now = DateTime.UtcNow; - - var requests = new[] + if (_ethusd == null) { - new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - Resolution.Minute, - false, - false, - DataNormalizationMode.Adjusted, - TickType.Trade) - }; - - var history = historyProvider.GetHistory(requests, TimeZones.Utc); - - foreach (var slice in history) - { - var bar = slice.Bars[symbol]; - - Log.Debug($"{bar.Time}: {bar}"); + TestGlobals.Initialize(); + _ethusd = Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken); } - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); - }; - - if (throwsException) - { - Assert.Throws(test); - } - else - { - Assert.DoesNotThrow(test); + return _ethusd; } } [Test] - [TestCaseSource(nameof(NoHistory))] - public void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period) + [TestCaseSource(nameof(ValidHistory))] + [TestCaseSource(nameof(InvalidHistory))] + public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool unsupported) { - TestDelegate test = () => + var brokerage = unsupported ? TestableKrakenBrokerage.Create() : (KrakenBrokerage)Brokerage; + + var now = DateTime.UtcNow; + var request = new HistoryRequest(now.Add(-period), + now, + typeof(TradeBar), + symbol, + resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + DateTimeZone.Utc, + Resolution.Minute, + false, + false, + DataNormalizationMode.Adjusted, + tickType); + + var history = brokerage.GetHistory(request)?.ToList(); + + if (unsupported) { - var brokerage = (KrakenBrokerage)Brokerage; - - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null,false, new DataPermissionManager(), null)); - - var now = DateTime.UtcNow; - - var requests = new[] - { - new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - Resolution.Minute, - false, - false, - DataNormalizationMode.Adjusted, - TickType.Trade) - }; + Assert.IsNull(history); + return; + } - var history = historyProvider.GetHistory(requests, TimeZones.Utc).ToList(); + Assert.IsNotNull(history); - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); - Assert.AreEqual(0, historyProvider.DataPointCount); - Assert.IsEmpty(history); - }; + foreach (var bar in history.Cast()) + { + Log.Debug($"{bar.Time}: {bar}"); + } - Assert.DoesNotThrow(test); + Log.Trace("Data points retrieved: " + history.Count); } private static TestCaseData[] ValidHistory @@ -129,40 +88,59 @@ private static TestCaseData[] ValidHistory return new[] { // valid - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken), Resolution.Tick, Time.OneHour, false), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken), Resolution.Minute, Time.OneDay, false), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken), Resolution.Hour, TimeSpan.FromDays(30), false), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken), Resolution.Daily, TimeSpan.FromDays(15), false), + new TestCaseData(ETHUSD, Resolution.Tick, TickType.Trade, Time.OneHour, false), + new TestCaseData(ETHUSD, Resolution.Minute, TickType.Trade, Time.OneDay, false), + new TestCaseData(ETHUSD, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(30), false), + new TestCaseData(ETHUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), false), }; } } - private static TestCaseData[] NoHistory + private static TestCaseData[] InvalidHistory { get { + TestGlobals.Initialize(); return new[] { - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.Crypto, Market.Kraken), Resolution.Second, Time.OneMinute), - - // invalid security type, no error, empty result" - new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15)), - - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15)), + // invalid symbol + new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.Kraken), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true), + + // invalid security type + new TestCaseData(Symbols.AAPL, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true), + + // invalid resolution + new TestCaseData(ETHUSD, Resolution.Second, TickType.Trade, TimeSpan.FromDays(15), true), + + // invalid tick type + new TestCaseData(ETHUSD, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(15), true), + new TestCaseData(ETHUSD, Resolution.Daily, TickType.OpenInterest, TimeSpan.FromDays(15), true), + + // invalid period + new TestCaseData(ETHUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(-15), true), + }; } } - private static TestCaseData[] InvalidHistory + private class TestableKrakenBrokerage : KrakenBrokerage { - get + public TestableKrakenBrokerage() { - return new[] - { - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" - new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.Kraken), Resolution.Daily, TimeSpan.FromDays(15), true) - }; + } + + public override void Connect() + { + } + + public static TestableKrakenBrokerage Create() + { + var brokerage = new TestableKrakenBrokerage(); + + var factory = new KrakenBrokerageFactory(); + brokerage.SetJob(new Packets.LiveNodePacket() { BrokerageData = factory.BrokerageData }); + + return brokerage; } } } diff --git a/QuantConnect.KrakenBrokerage.ToolBox/KrakenDataDownloader.cs b/QuantConnect.KrakenBrokerage.ToolBox/KrakenDataDownloader.cs index 93d5c33..15841ce 100644 --- a/QuantConnect.KrakenBrokerage.ToolBox/KrakenDataDownloader.cs +++ b/QuantConnect.KrakenBrokerage.ToolBox/KrakenDataDownloader.cs @@ -54,22 +54,6 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa var resolution = dataDownloaderGetParameters.Resolution; var startUtc = dataDownloaderGetParameters.StartUtc; var endUtc = dataDownloaderGetParameters.EndUtc; - var tickType = dataDownloaderGetParameters.TickType; - - if (tickType != TickType.Trade) - { - yield break; - } - - if (endUtc < startUtc) - { - throw new ArgumentException("The end date must be greater or equal than the start date."); - } - - if (!_symbolMapper.IsKnownLeanSymbol(symbol)) - { - throw new ArgumentException($"The ticker {symbol.Value} is not available in Kraken. Use Lean symbols for downloader (i.e BTCUSD, not XXBTZUSD)"); - } var historyRequest = new HistoryRequest( startUtc, @@ -85,10 +69,7 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa DataNormalizationMode.Adjusted, TickType.Trade); - foreach (var baseData in _brokerage.GetHistory(historyRequest)) - { - yield return baseData; - } + return _brokerage.GetHistory(historyRequest); } } diff --git a/QuantConnect.KrakenBrokerage.ToolBox/KrakenDownloaderProgram.cs b/QuantConnect.KrakenBrokerage.ToolBox/KrakenDownloaderProgram.cs index c2c0e14..ef1edea 100644 --- a/QuantConnect.KrakenBrokerage.ToolBox/KrakenDownloaderProgram.cs +++ b/QuantConnect.KrakenBrokerage.ToolBox/KrakenDownloaderProgram.cs @@ -53,8 +53,13 @@ public static void KrakenDownloader(IList tickers, string resolution, Da // Download data var pairObject = Symbol.Create(pair, SecurityType.Crypto, Market.Kraken); var data = downloader.Get(new DataDownloaderGetParameters(pairObject, castResolution == Resolution.Second ? Resolution.Tick : castResolution, startDate, endDate)); + if (data == null) + { + continue; + } + var bars = data.Cast().ToList(); - + // Write data var writer = new LeanDataWriter(castResolution, pairObject, dataDirectory); @@ -71,7 +76,7 @@ public static void KrakenDownloader(IList tickers, string resolution, Da Log.Error(err); } } - + /// /// Endpoint for downloading exchange info /// diff --git a/QuantConnect.KrakenBrokerage/KrakenBrokerage.DataQueueHandler.cs b/QuantConnect.KrakenBrokerage/KrakenBrokerage.DataQueueHandler.cs index 67c0deb..9ec4d82 100644 --- a/QuantConnect.KrakenBrokerage/KrakenBrokerage.DataQueueHandler.cs +++ b/QuantConnect.KrakenBrokerage/KrakenBrokerage.DataQueueHandler.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using QuantConnect.Brokerages.Kraken.Models; @@ -41,7 +40,7 @@ public partial class KrakenBrokerage /// Need Token to have access to auth wss /// private string WebsocketToken { get; set; } - + private readonly ConcurrentDictionary _orderBooks = new (); private string _orderBookChannel; @@ -95,11 +94,11 @@ private bool Subscribe(IWebSocket webSocket, Symbol symbol) }, subscription = new { - depth = _orderBookDepth, + depth = _orderBookDepth, name = "book" } }); - + WebsocketSend(webSocket, new { @event = "subscribe", @@ -136,7 +135,7 @@ private bool Unsubscribe(IWebSocket webSocket, Symbol symbol) name = "book" } }); - + WebsocketSend(webSocket, new { @event = "unsubscribe", @@ -156,9 +155,9 @@ private bool Unsubscribe(IWebSocket webSocket, Symbol symbol) private void WebsocketSend(IWebSocket webSocket, object data) { var message = JsonConvert.SerializeObject(data); - + _webSocketRateLimiter.WaitToProceed(); - + webSocket.Send(message); } @@ -196,9 +195,9 @@ private void OnDataMessage(WebSocketMessage webSocketMessage) else if (token is JArray jToken) { var channel = jToken[jToken.Count - 2].ToString(); - + var symbol = _symbolMapper.GetSymbolFromWebsocket(jToken.Last.ToString()); - + if (channel == _orderBookChannel) { if (jToken[1]["as"] != null || jToken[1]["bs"] != null) // snapshot @@ -209,7 +208,7 @@ private void OnDataMessage(WebSocketMessage webSocketMessage) { ProcessOrderBookUpdate(symbol, jToken[1]); } - + } else if (channel == "trade") { @@ -351,7 +350,7 @@ private void ProcessOrderBookUpdate(Symbol symbol, JToken book) throw; } } - + private void EmitQuoteTick(Symbol symbol, decimal bidPrice, decimal bidSize, decimal askPrice, decimal askSize) { try @@ -367,7 +366,7 @@ private void EmitQuoteTick(Symbol symbol, decimal bidPrice, decimal bidSize, dec BidSize = Math.Abs(bidSize), Exchange = Market.Kraken }; - + tick.SetValue(); EmitTick(tick); } @@ -392,11 +391,10 @@ public void EmitTick(Tick tick) private bool CanSubscribe(Symbol symbol) { - if (symbol.Value.Contains("UNIVERSE") || !_symbolMapper.IsKnownLeanSymbol(symbol) || symbol.ID.Market != Market.Kraken) - { - return false; - } - return true; + return !symbol.Value.Contains("UNIVERSE") && + symbol.SecurityType == SecurityType.Crypto && + symbol.ID.Market == Market.Kraken && + _symbolMapper.IsKnownLeanSymbol(symbol); } private void OnBestBidAskUpdated(object sender, BestBidAskUpdatedEventArgs e) @@ -409,7 +407,7 @@ private void OnBestBidAskUpdated(object sender, BestBidAskUpdatedEventArgs e) #region IDataQueueHandler /// - /// Set websocket token. Needs when subscribing to private feeds + /// Set websocket token. Needs when subscribing to private feeds /// private void SetWebsocketToken() { diff --git a/QuantConnect.KrakenBrokerage/KrakenBrokerage.cs b/QuantConnect.KrakenBrokerage/KrakenBrokerage.cs index 7bb5d6a..64a60af 100644 --- a/QuantConnect.KrakenBrokerage/KrakenBrokerage.cs +++ b/QuantConnect.KrakenBrokerage/KrakenBrokerage.cs @@ -62,6 +62,11 @@ public partial class KrakenBrokerage : BaseWebsocketsBrokerage, IDataQueueHandle private readonly ConcurrentDictionary _fills = new ConcurrentDictionary(); + private bool _loggedUnsupportedAssetForHistory; + private bool _loggedUnsupportedResolutionForHistory; + private bool _loggedUnsupportedTickTypeForHistory; + private bool _loggedInvalidDateRangeForHistory; + /// /// Constructor for brokerage /// @@ -387,7 +392,6 @@ public override bool CancelOrder(Order order) return true; } - /// /// Gets the history for the requested security /// @@ -395,25 +399,48 @@ public override bool CancelOrder(Order order) /// An enumerable of bars covering the span specified in the request public override IEnumerable GetHistory(HistoryRequest request) { + if (!CanSubscribe(request.Symbol)) + { + if (!_loggedUnsupportedAssetForHistory) + { + _loggedUnsupportedAssetForHistory = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "UnsupportedAssert", + $"Unsupported asset {request.Symbol} Make sure your asserts are of {SecurityType.Crypto} type only.")); + } + return null; + } + if (request.Resolution == Resolution.Second) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", - $"{request.Resolution} resolution is not supported, no history returned")); - yield break; + if (!_loggedUnsupportedResolutionForHistory) + { + _loggedUnsupportedResolutionForHistory = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", + $"{request.Resolution} resolution is not supported, no history returned")); + } + return null; } if (request.TickType != TickType.Trade) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", - $"{request.TickType} tick type not supported, no history returned")); - yield break; + if (!_loggedUnsupportedTickTypeForHistory) + { + _loggedUnsupportedTickTypeForHistory = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", + $"{request.TickType} tick type not supported, no history returned")); + } + return null; } - if (request.Symbol.SecurityType != SecurityType.Crypto) + if (request.StartTimeUtc >= request.EndTimeUtc) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType", - $"{request.Symbol.SecurityType} tick type not supported, no history returned. Use only {SecurityType.Crypto} type.")); - yield break; + if (!_loggedInvalidDateRangeForHistory) + { + _loggedInvalidDateRangeForHistory = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange", + "The history request start date must precede the end date. No history returned.")); + } + return null; } var period = request.Resolution.ToTimeSpan(); @@ -426,11 +453,9 @@ public override IEnumerable GetHistory(HistoryRequest request) var marketSymbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); - var enumerator = request.Resolution == Resolution.Tick ? GetTradeBars(request, marketSymbol) : GetOhlcBars(request, marketSymbol, period); - foreach (var baseData in enumerator) - { - yield return baseData; - } + return request.Resolution == Resolution.Tick + ? GetTradeBars(request, marketSymbol) + : GetOhlcBars(request, marketSymbol, period); } ///