Private cloud minor improvements (#8409)

* Private cloud tweaks

* Minor tweaks for optimization

Api/ApiConnection.cs

@@ -50,7 +50,7 @@ public ApiConnection(int userId, string token)
         {
             _token = token;
             _userId = userId.ToStringInvariant();
-            Client = new RestClient("https://www.quantconnect.com/api/v2/");
+            Client = new RestClient(Globals.Api);
         }
 
         /// <summary>
@@ -112,15 +112,15 @@ public async Task<Tuple<bool, T>> TryRequestAsync<T>(RestRequest request)
 
                 if (!restsharpResponse.IsSuccessful)
                 {
-                    Log.Error($"ApiConnect.TryRequest(): Content: {restsharpResponse.Content}");
+                    Log.Error($"ApiConnect.TryRequest({request.Resource}): Content: {restsharpResponse.Content}");
                 }
 
                 responseContent = restsharpResponse.Content;
                 result = JsonConvert.DeserializeObject<T>(responseContent, _jsonSettings);
 
                 if (result == null || !result.Success)
                 {
-                    Log.Debug($"ApiConnection.TryRequest(): Raw response: '{responseContent}'");
+                    Log.Debug($"ApiConnection.TryRequest({request.Resource}): Raw response: '{responseContent}'");
                     return new Tuple<bool, T>(false, result);
                 }
             }

Common/Api/Backtest.cs

@@ -18,6 +18,7 @@
 using QuantConnect.Statistics;
 using System.Collections.Generic;
 using QuantConnect.Optimizer.Parameters;
+using QuantConnect.Util;
 
 namespace QuantConnect.Api
 {
@@ -52,6 +53,16 @@ public class ResearchGuide
     /// </summary>
     public class BasicBacktest : RestResponse
     {
+        /// <summary>
+        /// Backtest error message
+        /// </summary>
+        public string Error { get; set; }
+
+        /// <summary>
+        /// Backtest error stacktrace
+        /// </summary>
+        public string Stacktrace { get; set; }
+
         /// <summary>
         /// Assigned backtest Id
         /// </summary>
@@ -70,6 +81,7 @@ public class BasicBacktest : RestResponse
         /// <summary>
         /// Backtest creation date and time
         /// </summary>
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
         public DateTime Created { get; set; }
 
         /// <summary>
@@ -103,7 +115,6 @@ public class BasicBacktest : RestResponse
     /// </summary>
     public class Backtest : BasicBacktest
     {
-
         /// <summary>
         /// Note on the backtest attached by the user
         /// </summary>
@@ -114,16 +125,6 @@ public class Backtest : BasicBacktest
         /// </summary>
         public bool Completed { get; set; }
 
-        /// <summary>
-        /// Backtest error message
-        /// </summary>
-        public string Error { get; set; }
-
-        /// <summary>
-        /// Backtest error stacktrace
-        /// </summary>
-        public string StackTrace { get; set; }
-
         /// <summary>
         /// Organization ID
         /// </summary>
@@ -185,6 +186,11 @@ public class Backtest : BasicBacktest
         /// </summary>
         public string NodeName { get; set; }
 
+        /// <summary>
+        /// The associated project id
+        /// </summary>
+        public int ProjectId { get; set; }
+
         /// <summary>
         /// End date of out of sample data
         /// </summary>

Common/Api/BaseOptimization.cs

@@ -19,6 +19,9 @@
 using QuantConnect.Optimizer.Objectives;
 using System.Collections.Generic;
 using QuantConnect.Optimizer.Parameters;
+using QuantConnect.Util;
+using Newtonsoft.Json.Converters;
+using Newtonsoft.Json.Serialization;
 
 namespace QuantConnect.Api
 {
@@ -45,6 +48,7 @@ public class BaseOptimization : RestResponse
         /// <summary>
         /// Status of the optimization
         /// </summary>
+        [JsonConverter(typeof(StringEnumConverter), converterParameters: typeof(CamelCaseNamingStrategy))]
         public OptimizationStatus Status { get; set; }
 
         /// <summary>
@@ -61,7 +65,8 @@ public class BaseOptimization : RestResponse
         /// <summary>
         /// End date of out of sample data
         /// </summary>
-        public DateTime OutOfSampleMaxEndDate { get; set; }
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
+        public DateTime? OutOfSampleMaxEndDate { get; set; }
 
         /// <summary>
         /// Parameters used in this optimization
@@ -82,6 +87,7 @@ public class OptimizationSummary: BaseOptimization
         /// <summary>
         /// Date when this optimization was created
         /// </summary>
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
         public DateTime Created { get; set; }
 
         /// <summary>

Common/Api/Optimization.cs

@@ -17,6 +17,7 @@
 using System.Collections.Generic;
 using Newtonsoft.Json;
 using QuantConnect.Optimizer.Objectives;
+using QuantConnect.Util;
 
 namespace QuantConnect.Api
 {
@@ -67,10 +68,11 @@ public class Optimization : BaseOptimization
         /// Optimization strategy
         /// </summary>
         public string Strategy { get; set; }
-        
+
         /// <summary>
         /// Optimization requested date and time
         /// </summary>
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
         public DateTime Requested { get; set; }
     }
 
@@ -94,5 +96,10 @@ public class OptimizationList : RestResponse
         /// Collection of summarized optimization objects
         /// </summary>
         public List<OptimizationSummary> Optimizations { get; set; }
+
+        /// <summary>
+        /// The optimization count
+        /// </summary>
+        public int Count => Optimizations?.Count ?? 0;
     }
 }

Common/Api/Project.cs

@@ -150,6 +150,12 @@ public class GridChart
         /// </summary>
         [JsonProperty(PropertyName = "sort")]
         public int Sort { get; set; }
+
+        /// <summary>
+        /// Optionally related definition
+        /// </summary>
+        [JsonProperty(PropertyName = "definition")]
+        public List<string> Definition { get; set; }
     }
 
     /// <summary>

Common/Globals.cs

@@ -33,7 +33,7 @@ static Globals()
         /// <summary>
         /// The base api url address to use
         /// </summary>
-        public static string Api { get; } = "https://www.quantconnect.com/api/v2/";
+        public static string Api { get; set; }
 
         /// <summary>
         /// The user Id
@@ -96,6 +96,7 @@ public static void Reset ()
             ProjectId = Config.GetInt("project-id");
             UserToken = Config.Get("api-access-token");
             OrganizationID = Config.Get("job-organization-id");
+            Api = Config.Get("api-url", "https://www.quantconnect.com/api/v2/");
             ResultsDestinationFolder = Config.Get("results-destination-folder", Directory.GetCurrentDirectory());
         }
 

Common/Isolator.cs

@@ -100,6 +100,12 @@ private bool MonitorTask(Task task,
             memoryCap *= 1024 * 1024;
             var spikeLimit = memoryCap*2;
 
+            if (memoryCap <= 0)
+            {
+                memoryCap = int.MaxValue;
+                spikeLimit = int.MaxValue;
+            }
+
             while (!task.IsCompleted && !CancellationTokenSource.IsCancellationRequested && utcNow < end)
             {
                 // if over 80% allocation force GC then sample

Common/Optimizer/Objectives/Objective.cs

@@ -83,12 +83,16 @@ protected Objective(string target, decimal? targetValue)
         /// <remarks>For <see cref="Objectives.Target"/> if defined and backtest complies with the targets then finish optimization</remarks>
         /// <remarks>For <see cref="Constraint"/> non optional, the value of the target constraint</remarks>
         [JsonProperty("target-value")]
-        private decimal? OldTargetValue
+        public decimal? OldTargetValue
         {
             set
             {
                 TargetValue = value;
             }
+            get
+            {
+                return TargetValue;
+            }
         }
         #endregion
     }

Common/Optimizer/Parameters/OptimizationParameterJsonConverter.cs

@@ -60,7 +60,7 @@ JsonSerializer serializer
             )
         {
             JObject token = JObject.Load(reader);
-            var parameterName = token.GetValue("name", StringComparison.OrdinalIgnoreCase)?.Value<string>();
+            var parameterName = (token.GetValue("name", StringComparison.OrdinalIgnoreCase) ?? token.GetValue("key", StringComparison.OrdinalIgnoreCase))?.Value<string>();
             if (string.IsNullOrEmpty(parameterName))
             {
                 throw new ArgumentException(Messages.OptimizationParameterJsonConverter.OptimizationParameterNotSpecified);
@@ -70,11 +70,7 @@ JsonSerializer serializer
             JToken minToken;
             JToken maxToken;
             OptimizationParameter optimizationParameter = null;
-            if (token.TryGetValue("value", StringComparison.OrdinalIgnoreCase, out value))
-            {
-                optimizationParameter = new StaticOptimizationParameter(parameterName, value.Value<string>());
-            }
-            else if (token.TryGetValue("min", StringComparison.OrdinalIgnoreCase, out minToken) &&
+            if (token.TryGetValue("min", StringComparison.OrdinalIgnoreCase, out minToken) &&
                 token.TryGetValue("max", StringComparison.OrdinalIgnoreCase, out maxToken))
             {
                 var stepToken = token.GetValue("step", StringComparison.OrdinalIgnoreCase)?.Value<decimal>();
@@ -104,6 +100,10 @@ JsonSerializer serializer
                         maxToken.Value<decimal>());
                 }
             }
+            else if(token.TryGetValue("value", StringComparison.OrdinalIgnoreCase, out value))
+            {
+                optimizationParameter = new StaticOptimizationParameter(parameterName, value.Value<string>());
+            }
 
             if (optimizationParameter == null)
             {

Common/Optimizer/Parameters/ParameterSet.cs

@@ -46,10 +46,10 @@ public class ParameterSet
         /// </summary>
         /// <param name="id">Unique identifier</param>
         /// <param name="value">Combination of optimization parameters</param>
-        public ParameterSet(int id, Dictionary<string, string> value)
+        public ParameterSet(int id, IReadOnlyDictionary<string, string> value)
         {
             Id = id;
-            Value = value?.ToReadOnlyDictionary();
+            Value = value;
         }
 
         /// <summary>

Common/Packets/AlgorithmNodePacket.cs

@@ -124,7 +124,7 @@ public string AlgorithmId
         /// The maximum amount of RAM (in MB) this algorithm is allowed to utilize
         /// </summary>
         public int RamAllocation {
-            get { return Controls.RamAllocation; }
+            get { return Controls?.RamAllocation ?? 0; }
         }
 
         /// <summary>

Common/Packets/Packet.cs

@@ -194,5 +194,8 @@ public enum PacketType
 
         /// Research job packet
         ResearchNode,
+
+        /// Organization update
+        OrganizationUpdate,
     }
 }

Engine/Results/BacktestingResultHandler.cs

@@ -90,6 +90,10 @@ public override void Initialize(ResultHandlerInitializeParameters parameters)
             _projectId = _job.ProjectId;
             if (_job == null) throw new Exception("BacktestingResultHandler.Constructor(): Submitted Job type invalid.");
             base.Initialize(parameters);
+            if (!string.IsNullOrEmpty(_job.OptimizationId))
+            {
+                State["OptimizationId"] = _job.OptimizationId;
+            }
         }
 
         /// <summary>

Engine/Results/BaseResultsHandler.cs

@@ -90,6 +90,11 @@ public abstract class BaseResultsHandler
         /// </summary>
         public const string PortfolioMarginKey = "Portfolio Margin";
 
+        /// <summary>
+        /// String message saying: Portfolio Margin
+        /// </summary>
+        public const string AssetsSalesVolumeKey = "Assets Sales Volume";
+
         /// <summary>
         /// The main loop update interval
         /// </summary>
@@ -720,7 +725,7 @@ protected virtual void SampleSalesVolume(DateTime time)
             foreach (var holding in Algorithm.Portfolio.Values.Where(y => y.TotalSaleVolume != 0)
                 .OrderByDescending(x => x.TotalSaleVolume).Take(30))
             {
-                Sample("Assets Sales Volume", $"{holding.Symbol.Value}", 0, SeriesType.Treemap, new ChartPoint(time, holding.TotalSaleVolume),
+                Sample(AssetsSalesVolumeKey, $"{holding.Symbol.Value}", 0, SeriesType.Treemap, new ChartPoint(time, holding.TotalSaleVolume),
                     AlgorithmCurrencySymbol);
             }
         }

Engine/Setup/BacktestingSetupHandler.cs

@@ -132,9 +132,7 @@ public bool Setup(SetupHandlerParameters parameters)
                 throw new ArgumentException("Expected BacktestNodePacket but received " + parameters.AlgorithmNodePacket.GetType().Name);
             }
 
-            Log.Trace($"BacktestingSetupHandler.Setup(): Setting up job: UID: {job.UserId.ToStringInvariant()}, " +
-                $"PID: {job.ProjectId.ToStringInvariant()}, Version: {job.Version}, Source: {job.RequestSource}"
-            );
+            BaseSetupHandler.Setup(parameters);
 
             if (algorithm == null)
             {
@@ -164,7 +162,6 @@ public bool Setup(SetupHandlerParameters parameters)
 
                     //Algorithm is backtesting, not live:
                     algorithm.SetAlgorithmMode(job.AlgorithmMode);
-                    algorithm.SetDeploymentTarget(job.DeploymentTarget);
 
                     //Set the source impl for the event scheduling
                     algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);

Engine/Setup/BaseSetupHandler.cs

@@ -45,6 +45,24 @@ public static class BaseSetupHandler
         /// </summary>
         public static TimeSpan AlgorithmCreationTimeout { get; } = TimeSpan.FromSeconds(Config.GetDouble("algorithm-creation-timeout", 90));
 
+        /// <summary>
+        /// Primary entry point to setup a new algorithm
+        /// </summary>
+        /// <param name="parameters">The parameters object to use</param>
+        /// <returns>True on successfully setting up the algorithm state, or false on error.</returns>
+        public static bool Setup(SetupHandlerParameters parameters)
+        {
+            var algorithm = parameters.Algorithm;
+            var job = parameters.AlgorithmNodePacket;
+
+            algorithm?.SetDeploymentTarget(job.DeploymentTarget);
+
+            Log.Trace($"BaseSetupHandler.Setup({job.DeploymentTarget}): UID: {job.UserId.ToStringInvariant()}, " +
+                $"PID: {job.ProjectId.ToStringInvariant()}, Version: {job.Version}, Source: {job.RequestSource}"
+            );
+            return true;
+        }
+
         /// <summary>
         /// Will first check and add all the required conversion rate securities
         /// and later will seed an initial value to them.

Engine/Setup/BrokerageSetupHandler.cs

@@ -163,6 +163,7 @@ public bool Setup(SetupHandlerParameters parameters)
                 return false;
             }
 
+            BaseSetupHandler.Setup(parameters);
 
             // attach to the message event to relay brokerage specific initialization messages
             EventHandler<BrokerageMessageEvent> brokerageOnMessage = (sender, args) =>
@@ -243,7 +244,6 @@ public bool Setup(SetupHandlerParameters parameters)
 
                         //Algorithm is live, not backtesting:
                         algorithm.SetAlgorithmMode(liveJob.AlgorithmMode);
-                        algorithm.SetDeploymentTarget(liveJob.DeploymentTarget);
 
                         //Initialize the algorithm's starting date
                         algorithm.SetDateTime(DateTime.UtcNow);