Feature .net 5 (#5505)

* Update projects to use .NET 5.0, the successor to .NET Core

* Fix ambiguous errors. Add IBAutomator net5

* Remove FXCM

* Upgrade IBAutomater to v1.0.51

ignored, and an empty message aborts the commit.

* Fix rebase

- Fix ambiguous Index
- Remove StrategyCapacity.cs
- Update System.Threading.Tasks.Extensionsy

* Remove unrequired references

* Fixes

- Travis will use dotnet, not nunit nor mono
- Remove mono from foundation image
- Fix python setup in research
- Fix unit tests

* Don't call ReadKey when input is redirected

* Fix ConsoleLeanOptimizer

* Research fixes

* Update comment

* Add vsdbg to Dockerfile

* Fixes

- Revert dockerfile FROM custom changes
- Adjust and fix regression algorithms
   - Option assignment will be deterministic in the order
   - 'Rolling Averaged Population' is calculated using doubles, updating
     expected values.
- Update readme, removing references to mono
- Add missing Py.Gil lock

* Replace ICSharp with .NET Interactive

* Fixes after rebase

* CSharp research fixes

- Adding new Initialize.csx that pre loads all assemblies
- Adjusting template research file
- Moving steps in dockerfilejupyter
- Fix unit tests and regression tests after rebase

Co-authored-by: Gerardo Salazar <[email protected]>
Co-authored-by: Stefano Raggi <[email protected]>
Co-authored-by: Jasper van Merle <[email protected]>

.github/workflows/gh-actions.yml

@@ -9,22 +9,17 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     container:
         image: quantconnect/lean:foundation
     steps:
       - uses: actions/checkout@v2
 
-      - name: Restore nuget dependencies
-        run: |
-          nuget restore QuantConnect.Lean.sln -v quiet
-          nuget install NUnit.Runners -Version 3.11.1 -OutputDirectory testrunner
-
       - name: Build
-        run: msbuild /p:Configuration=Release /p:VbcToolExe=vbnc.exe /v:quiet /p:WarningLevel=1 QuantConnect.Lean.sln
+        run: dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.Lean.sln
 
       - name: Run Tests
-        run: mono ./testrunner/NUnit.ConsoleRunner.3.11.1/tools/nunit3-console.exe ./Tests/bin/Release/QuantConnect.Tests.dll --where "cat != TravisExclude" --labels=Off --params:log-handler=ConsoleErrorLogHandler
+        run: dotnet test ./Tests/bin/Release/QuantConnect.Tests.dll --filter TestCategory!=TravisExclude -- TestRunParameters.Parameter\(name=\"log-handler\", value=\"ConsoleErrorLogHandler\"\)
 
       - name: Generate & Publish python stubs
         if: startsWith(github.ref, 'refs/tags/')

.travis.yml

@@ -1,9 +1,8 @@
-sudo: required
 language: csharp
+mono: none
 dotnet: 5.0
-mono:
-  - 5.12.0
-solution: QuantConnect.Lean.sln
+os: linux
+dist: focal
 before_install:
   - export PATH="$HOME/miniconda3/bin:$PATH"
   - export PYTHONNET_PYDLL="$HOME/miniconda3/lib/libpython3.6m.so"
@@ -18,9 +17,7 @@ before_install:
   - conda install -y cython=0.29.15
   - conda install -y scipy=1.4.1
   - conda install -y wrapt=1.12.1
-install:
-  - nuget install NUnit.Runners -Version 3.11.1 -OutputDirectory testrunner
 script:
   - dotnet nuget add source $TRAVIS_BUILD_DIR/LocalPackages
-  - dotnet build /p:Configuration=Release /p:VbcToolExe=vbnc.exe /v:quiet /p:WarningLevel=1 QuantConnect.Lean.sln
-  - mono ./testrunner/NUnit.ConsoleRunner.3.11.1/tools/nunit3-console.exe ./Tests/bin/Release/QuantConnect.Tests.dll --where "cat != TravisExclude" --labels=Off --params:log-handler=ConsoleErrorLogHandler
+  - dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.Lean.sln
+  - dotnet test ./Tests/bin/Release/QuantConnect.Tests.dll --filter TestCategory!=TravisExclude -- TestRunParameters.Parameter\(name=\"log-handler\", value=\"ConsoleErrorLogHandler\"\)

Algorithm.CSharp/AddRemoveOptionUniverseRegressionAlgorithm.cs

@@ -79,19 +79,19 @@ public override void OnData(Slice data)
                 // things like manually added, auto added, internal, and any other boolean state we need to track against a single security)
                 throw new Exception("The underlying equity data should NEVER be removed in this algorithm because it was manually added");
             }
-            if (_expectedSecurities.AreDifferent(LinqExtensions.ToHashSet(Securities.Keys)))
+            if (_expectedSecurities.AreDifferent(Securities.Keys.ToHashSet()))
             {
                 var expected = string.Join(Environment.NewLine, _expectedSecurities.OrderBy(s => s.ToString()));
                 var actual = string.Join(Environment.NewLine, Securities.Keys.OrderBy(s => s.ToString()));
                 throw new Exception($"{Time}:: Detected differences in expected and actual securities{Environment.NewLine}Expected:{Environment.NewLine}{expected}{Environment.NewLine}Actual:{Environment.NewLine}{actual}");
             }
-            if (_expectedUniverses.AreDifferent(LinqExtensions.ToHashSet(UniverseManager.Keys)))
+            if (_expectedUniverses.AreDifferent(UniverseManager.Keys.ToHashSet()))
             {
                 var expected = string.Join(Environment.NewLine, _expectedUniverses.OrderBy(s => s.ToString()));
                 var actual = string.Join(Environment.NewLine, UniverseManager.Keys.OrderBy(s => s.ToString()));
                 throw new Exception($"{Time}:: Detected differences in expected and actual universes{Environment.NewLine}Expected:{Environment.NewLine}{expected}{Environment.NewLine}Actual:{Environment.NewLine}{actual}");
             }
-            if (_expectedData.AreDifferent(LinqExtensions.ToHashSet(data.Keys)))
+            if (_expectedData.AreDifferent(data.Keys.ToHashSet()))
             {
                 var expected = string.Join(Environment.NewLine, _expectedData.OrderBy(s => s.ToString()));
                 var actual = string.Join(Environment.NewLine, data.Keys.OrderBy(s => s.ToString()));
@@ -183,7 +183,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
                 if (changes.RemovedSecurities
                     .Where(x => x.Symbol.SecurityType == SecurityType.Option)
                     .ToHashSet(s => s.Symbol)
-                    .AreDifferent(LinqExtensions.ToHashSet(_expectedContracts)))
+                    .AreDifferent(_expectedContracts.ToHashSet()))
                 {
                     throw new Exception("Expected removed securities to equal expected contracts added");
                 }

Algorithm.CSharp/CoarseFineOptionUniverseChainRegressionAlgorithm.cs

@@ -151,18 +151,18 @@ public override void OnEndOfAlgorithm()
             {"Drawdown", "0.500%"},
             {"Expectancy", "1.393"},
             {"Net Profit", "32.840%"},
-            {"Sharpe Ratio", "7.14272222483913E+15"},
+            {"Sharpe Ratio", "7142722224839133"},
             {"Probabilistic Sharpe Ratio", "0%"},
             {"Loss Rate", "83%"},
             {"Win Rate", "17%"},
             {"Profit-Loss Ratio", "13.36"},
-            {"Alpha", "2.59468989671647E+16"},
+            {"Alpha", "25946898967164744"},
             {"Beta", "66.241"},
             {"Annual Standard Deviation", "3.633"},
             {"Annual Variance", "13.196"},
-            {"Information Ratio", "7.25220453625048E+15"},
+            {"Information Ratio", "7252204536250480"},
             {"Tracking Error", "3.578"},
-            {"Treynor Ratio", "391705233723350"},
+            {"Treynor Ratio", "391705233723349.5"},
             {"Total Fees", "$13.00"},
             {"Estimated Strategy Capacity", "$3000000.00"},
             {"Fitness Score", "0.232"},

Algorithm.CSharp/FreePortfolioValueRegressionAlgorithm.cs

@@ -117,8 +117,8 @@ public override void OnOrderEvent(OrderEvent orderEvent)
             {"Mean Population Estimated Insight Value", "$54597.4919"},
             {"Mean Population Direction", "43.4499%"},
             {"Mean Population Magnitude", "43.4499%"},
-            {"Rolling Averaged Population Direction", "48.7105%"},
-            {"Rolling Averaged Population Magnitude", "48.7105%"},
+            {"Rolling Averaged Population Direction", "48.5717%"},
+            {"Rolling Averaged Population Magnitude", "48.5717%"},
             {"OrderListHash", "03cc0ad5b1c4b7803b2e9483da1d7543"}
         };
     }

Algorithm.CSharp/LongOnlyAlphaStreamAlgorithm.cs

@@ -136,7 +136,7 @@ public override void OnOrderEvent(OrderEvent orderEvent)
             {"Mean Population Estimated Insight Value", "$1443.9623"},
             {"Mean Population Direction", "62.5%"},
             {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "73.0394%"},
+            {"Rolling Averaged Population Direction", "73.1163%"},
             {"Rolling Averaged Population Magnitude", "0%"},
             {"OrderListHash", "79a973155c0106b60249931daa89c54b"}
         };

Algorithm.CSharp/OptionAssignmentRegressionAlgorithm.cs

@@ -117,7 +117,7 @@ public override void OnData(Slice data)
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "c5a34ac5f125ba7137b010820360f5bd"}
+            {"OrderListHash", "58557574cf0489dd38fb37768f509ca1"}
         };
     }
 }

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -4,7 +4,7 @@
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
     <RootNamespace>QuantConnect.Algorithm.CSharp</RootNamespace>
     <AssemblyName>QuantConnect.Algorithm.CSharp</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <LangVersion>6</LangVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <OutputPath>bin\$(Configuration)\</OutputPath>
@@ -63,10 +63,6 @@
     <PackageReference Include="NodaTime" Version="3.0.5" />
     <PackageReference Include="R.NET" Version="1.9.0" />
   </ItemGroup>
-  <ItemGroup>
-    <Reference Include="System.Data.DataSetExtensions" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>

Algorithm.CSharp/UniverseUnchangedRegressionAlgorithm.cs

@@ -138,8 +138,8 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
             {"Mean Population Estimated Insight Value", "$-5454.712"},
             {"Mean Population Direction", "30%"},
             {"Mean Population Magnitude", "30%"},
-            {"Rolling Averaged Population Direction", "42.9591%"},
-            {"Rolling Averaged Population Magnitude", "42.9591%"},
+            {"Rolling Averaged Population Direction", "42.9939%"},
+            {"Rolling Averaged Population Magnitude", "42.9939%"},
             {"OrderListHash", "1480e456536d63f95d6b06bfaffe7eaa"}
         };
     }

Algorithm.Framework/Portfolio/ReturnsSymbolData.cs

@@ -17,7 +17,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Indicators;
-using QuantConnect.Util;
 
 namespace QuantConnect.Algorithm.Framework.Portfolio
 {
@@ -146,4 +145,4 @@ public static class ReturnsSymbolDataExtensions
                 .ToArray());
         }
     }
-}
+}

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -4,7 +4,7 @@
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
     <RootNamespace>QuantConnect.Algorithm.Framework</RootNamespace>
     <AssemblyName>QuantConnect.Algorithm.Framework</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <OutputPath>bin\$(Configuration)\</OutputPath>
     <AnalysisMode>AllEnabledByDefault</AnalysisMode>
@@ -54,10 +54,6 @@
     </PackageReference>
     <PackageReference Include="NodaTime" Version="3.0.5" />
   </ItemGroup>
-  <ItemGroup>
-    <Reference Include="System.Data.DataSetExtensions" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -4,7 +4,7 @@
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
     <RootNamespace>QuantConnect.Algorithm.Python</RootNamespace>
     <AssemblyName>QuantConnect.Algorithm.Python</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <LangVersion>6</LangVersion>
     <AnalysisMode>AllEnabledByDefault</AnalysisMode>
     <OutputPath>bin\$(Configuration)\</OutputPath>

Algorithm/QCAlgorithm.cs

@@ -45,8 +45,8 @@
 using QuantConnect.Algorithm.Framework.Risk;
 using QuantConnect.Algorithm.Framework.Selection;
 using QuantConnect.Algorithm.Selection;
-using QuantConnect.Securities.Index;
 using QuantConnect.Storage;
+using Index = QuantConnect.Securities.Index.Index;
 
 namespace QuantConnect.Algorithm
 {

Algorithm/QuantConnect.Algorithm.csproj

@@ -5,7 +5,7 @@
     <ProductVersion>8.0.30703</ProductVersion>
     <RootNamespace>QuantConnect.Algorithm</RootNamespace>
     <AssemblyName>QuantConnect.Algorithm</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
     <LangVersion>6</LangVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
@@ -64,10 +64,6 @@
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="NodaTime" Version="3.0.5" />
-
-  </ItemGroup>
-  <ItemGroup>
-    <Reference Include="Microsoft.CSharp" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -4,7 +4,7 @@
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
     <RootNamespace>QuantConnect.AlgorithmFactory</RootNamespace>
     <AssemblyName>QuantConnect.AlgorithmFactory</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <LangVersion>6</LangVersion>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <OutputPath>bin\$(Configuration)\</OutputPath>
@@ -48,10 +48,6 @@
     </PackageReference>
     <PackageReference Include="NodaTime" Version="3.0.5" />
   </ItemGroup>
-  <ItemGroup>
-    <Reference Include="System.Data.DataSetExtensions" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>

Api/QuantConnect.Api.csproj

@@ -5,7 +5,7 @@
     <ProductVersion>8.0.30703</ProductVersion>
     <RootNamespace>QuantConnect.Api</RootNamespace>
     <AssemblyName>QuantConnect.Api</AssemblyName>
-    <TargetFramework>net462</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
     <LangVersion>6</LangVersion>
@@ -58,12 +58,6 @@
     <PackageReference Include="RestSharp" Version="106.6.10" />
     <PackageReference Include="SharpZipLib" Version="1.2.0" />
   </ItemGroup>
-  <ItemGroup>
-    <Reference Include="System.Configuration" />
-    <Reference Include="System.Web" />
-    <Reference Include="System.Data.DataSetExtensions" />
-    <Reference Include="Microsoft.CSharp" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>

Brokerages/Backtesting/BasicOptionAssignmentSimulation.cs

@@ -136,7 +136,7 @@ public void SimulateMarketConditions(IBrokerage brokerage, IAlgorithm algorithm)
                 return result;
             };
 
-            algorithm.Securities
+            foreach(var pair in algorithm.Securities
                 // we take only options that expire soon
                 .Where(x => x.Key.ID.SecurityType.IsOption() &&
                             ((x.Key.ID.OptionStyle == OptionStyle.American && x.Key.ID.Date - algorithm.UtcTime <= _priorExpiration) ||
@@ -150,9 +150,11 @@ public void SimulateMarketConditions(IBrokerage brokerage, IAlgorithm algorithm)
                         (OptionHolding)x.Value.Holdings,
                         algorithm.Securities[x.Value.Symbol.Underlying],
                         algorithm.Portfolio.CashBook) > 0.0m)
-                .ToList()
+                .OrderBy(pair => pair.Key))
+            {
                 // we exercise options with positive expected P/L (over basic sale of option)
-                .ForEach(x => backtestingBrokerage.ActivateOptionAssignment((Option)x.Value, (int)((OptionHolding)x.Value.Holdings).AbsoluteQuantity));
+                backtestingBrokerage.ActivateOptionAssignment((Option) pair.Value, (int) ((OptionHolding) pair.Value.Holdings).AbsoluteQuantity);
+            }
 
         }