Throughput tests

mhoeger · Oct 10, 2018 · b5ad684 · b5ad684
1 parent c7ebbb2
commit b5ad684
Show file tree

Hide file tree

Showing 44 changed files with 1,756 additions and 92 deletions.
diff --git a/WebJobs.Script.Tests.E2E.sln b/WebJobs.Script.Tests.E2E.sln
@@ -5,7 +5,12 @@ VisualStudioVersion = 15.1.26403.7
 MinimumVisualStudioVersion = 15.0.0.0
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Script.Tests.E2E", "test\WebJobs.Script.Tests.E2E\WebJobs.Script.Tests.E2E.csproj", "{A782A37B-5738-4250-ADA5-53A0AC441BA8}"
 EndProject
+Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "WebJobs.Script.Tests.E2E.Shared", "test\WebJobs.Script.Tests.E2E.Shared\WebJobs.Script.Tests.E2E.Shared.shproj", "{82D61060-8774-491C-9D73-9798A34C2399}"
+EndProject
 Global
+	GlobalSection(SharedMSBuildProjectFiles) = preSolution
+		test\WebJobs.Script.Tests.E2E.Shared\WebJobs.Script.Tests.E2E.Shared.projitems*{82d61060-8774-491c-9d73-9798a34c2399}*SharedItemsImports = 13
+	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Release|Any CPU = Release|Any CPU

diff --git a/WebJobs.Script.Tests.Perf.sln b/WebJobs.Script.Tests.Perf.sln
@@ -0,0 +1,36 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.28010.2019
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Script.Tests.Perf", "test\WebJobs.Script.Tests.Perf\WebJobs.Script.Tests.Perf.csproj", "{AE84CBCF-F722-404F-AE41-61696CDAA2EC}"
+EndProject
+Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "WebJobs.Script.Tests.E2E.Shared", "test\WebJobs.Script.Tests.E2E.Shared\WebJobs.Script.Tests.E2E.Shared.shproj", "{82D61060-8774-491C-9D73-9798A34C2399}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebJobs.Script.Tests.Perf.Dashboard", "test\WebJobs.Script.Tests.Perf.Dashboard\WebJobs.Script.Tests.Perf.Dashboard.csproj", "{C20F06BF-554C-4707-9D09-4D55D20BEE24}"
+EndProject
+Global
+	GlobalSection(SharedMSBuildProjectFiles) = preSolution
+		test\WebJobs.Script.Tests.E2E.Shared\WebJobs.Script.Tests.E2E.Shared.projitems*{82d61060-8774-491c-9d73-9798a34c2399}*SharedItemsImports = 13
+	EndGlobalSection
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{AE84CBCF-F722-404F-AE41-61696CDAA2EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AE84CBCF-F722-404F-AE41-61696CDAA2EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AE84CBCF-F722-404F-AE41-61696CDAA2EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AE84CBCF-F722-404F-AE41-61696CDAA2EC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C20F06BF-554C-4707-9D09-4D55D20BEE24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C20F06BF-554C-4707-9D09-4D55D20BEE24}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C20F06BF-554C-4707-9D09-4D55D20BEE24}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C20F06BF-554C-4707-9D09-4D55D20BEE24}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {31B134C9-3A01-402E-8868-274A875F60E5}
+	EndGlobalSection
+EndGlobal
diff --git a/...ipt.Tests.E2E/ArmAuthenticationHelpers.cs → ...ts.E2E.Shared/ArmAuthenticationHelpers.cs b/...ipt.Tests.E2E/ArmAuthenticationHelpers.cs → ...ts.E2E.Shared/ArmAuthenticationHelpers.cs
@@ -7,7 +7,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public static class ArmAuthenticationHelpers
     {

diff --git a/test/WebJobs.Script.Tests.E2E/Constants.cs → ...Jobs.Script.Tests.E2E.Shared/Constants.cs b/test/WebJobs.Script.Tests.E2E/Constants.cs → ...Jobs.Script.Tests.E2E.Shared/Constants.cs
@@ -2,7 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class Constants
     {
@@ -18,5 +18,8 @@ public class Constants
         public const string TargetSitePublishingPasswordSettingName = "AzureWebJobsTargetSitePublishingPassword";
         public const string RuntimeExtensionPackageUrlSettingName = "AzureWebjobsRuntimePrivateExtensionPackageUrl";
         public const string ServiceBusKey = "AzureWebJobsServiceBus";
+        public const string TargetSiteMasterKey = "AzureWebJobsTargetSiteMasterKey";
+        public const string TargetSiteFunctionKey = "AzureWebJobsTargetSiteFunctionKey";
+        public const string VM = "AzureWebJobsVM";
     }
 }
diff --git a/test/WebJobs.Script.Tests.E2E/Extensions.cs → ...obs.Script.Tests.E2E.Shared/Extensions.cs b/test/WebJobs.Script.Tests.E2E/Extensions.cs → ...obs.Script.Tests.E2E.Shared/Extensions.cs
@@ -3,7 +3,7 @@
 using System.Net.Http.Headers;
 using System.Threading.Tasks;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public static class HttpClientExtensions
     {

diff --git a/test/WebJobs.Script.Tests.E2E/Function.cs → ...bJobs.Script.Tests.E2E.Shared/Function.cs b/test/WebJobs.Script.Tests.E2E/Function.cs → ...bJobs.Script.Tests.E2E.Shared/Function.cs
@@ -3,7 +3,7 @@
 
 using Newtonsoft.Json;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class Function
     {

diff --git a/...bs.Script.Tests.E2E/FunctionAppFixture.cs → ...pt.Tests.E2E.Shared/FunctionAppFixture.cs b/...bs.Script.Tests.E2E/FunctionAppFixture.cs → ...pt.Tests.E2E.Shared/FunctionAppFixture.cs
@@ -17,7 +17,7 @@
 using Newtonsoft.Json.Linq;
 using Xunit;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class FunctionAppFixture : IDisposable
     {
@@ -33,9 +33,21 @@ public FunctionAppFixture()
 
             _assert = new TrackedAssert(Telemetry);
 
-            FunctionDefaultKey = Guid.NewGuid().ToString().ToLower();
+            FunctionDefaultKey = Settings.SiteFunctionKey ?? Guid.NewGuid().ToString().ToLower();
 
-            Initialize().Wait();
+            for (int i = 0; i < 3; i++)
+            {
+                try
+                {
+                    Initialize().Wait();
+                    break;
+                }
+                catch
+                {
+                    // Best effort.
+                    System.Threading.Thread.Sleep(5000);
+                }
+            }
         }
 
         public TrackedAssert Assert => _assert;
@@ -54,7 +66,7 @@ private async Task Initialize()
             _logger.LogInformation("Initializing environment...");
 
             _kuduClient = new KuduClient($"https://{Settings.SiteName}.scm.azurewebsites.net", Settings.SitePublishingUser, Settings.SitePublishingPassword);
-            FunctionAppMasterKey = await _kuduClient.GetFunctionsMasterKey();
+            FunctionAppMasterKey = Settings.SiteMasterKey ?? await _kuduClient.GetFunctionsMasterKey();
 
             // to run tests against currently deployed site, skip
             // this step (can take over 2 minutes)
@@ -136,7 +148,7 @@ private async Task WaitForSite()
                     // doesn't currently start the host.
                     // var result = await client.GetAsync($"{Settings.SiteBaseAddress}");
                     var functions = await _kuduClient.GetFunctions();
-                    var result = await client.GetAsync($"{Settings.SiteBaseAddress}/admin/functions/{functions.First().Name}/status?code={FunctionAppMasterKey}");                    
+                    var result = await client.GetAsync($"{Settings.SiteBaseAddress}/admin/functions/{functions.First().Name}/status?code={FunctionAppMasterKey}");
                     statusCode = result.StatusCode;
                     if (statusCode == HttpStatusCode.OK)
                     {
@@ -148,10 +160,25 @@ private async Task WaitForSite()
 
             }
 
-            throw new InvalidOperationException("Wait for site timeout: {delay.TotalSeconds * 5} seconds.");
+            throw new InvalidOperationException($"Wait for site timeout: {delay.TotalSeconds * 5} seconds.");
 
         }
 
+        private async Task CheckVersionsMatch()
+        {
+            using (var client = new HttpClient())
+            {
+                HttpResponseMessage response = await client.GetAsync($"/admin/host/status?code={FunctionAppMasterKey}");
+
+                var status = await response.Content.ReadAsAsync<dynamic>();
+
+                if (!string.Equals(Settings.RuntimeVersion, status.version.ToString()))
+                {
+                    throw new InvalidOperationException($"Versions aren't match. Actual version: {status.version.ToString()}, Package version: {Settings.RuntimeVersion}");
+                }
+            }
+        }
+
         private async Task AddSettings()
         {
             _logger.LogInformation("Updating app settings...");
@@ -171,7 +198,7 @@ private async Task UpdateSiteContents()
             string filePath = Path.ChangeExtension(Path.GetTempFileName(), "zip");
             ZipFile.CreateFromDirectory(Path.Combine(Environment.CurrentDirectory, "Functions"), filePath);
 
-            await _kuduClient.UploadZip("site", filePath);
+            await _kuduClient.DeployZip(filePath);
 
             _logger.LogInformation("Site contents updated");
         }

diff --git a/test/WebJobs.Script.Tests.E2E/KuduClient.cs → ...obs.Script.Tests.E2E.Shared/KuduClient.cs b/test/WebJobs.Script.Tests.E2E/KuduClient.cs → ...obs.Script.Tests.E2E.Shared/KuduClient.cs
@@ -10,7 +10,7 @@
 using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class KuduClient : IDisposable
     {
@@ -34,7 +34,15 @@ public KuduClient(string kuduUri, string userName, string password)
 
         public async Task DeleteDirectory(string path, bool recursive)
         {
-            await _client.DeleteAsync($"api/vfs/{path}/?recursive={recursive}");
+            HttpResponseMessage response = await _client.DeleteAsync($"api/vfs/{path}/?recursive={recursive}");
+            response.EnsureSuccessStatusCode();
+        }
+
+        public async Task DeployZip(string zipPath)
+        {
+            var content = new StreamContent(File.Open(zipPath, FileMode.Open));
+            HttpResponseMessage response = await _client.PostAsync("/api/zipdeploy", content);
+            response.EnsureSuccessStatusCode();
         }
 
         public async Task UploadZip(string sitePath, string zipPath)
@@ -59,7 +67,13 @@ public async Task<string> GetFunctionsMasterKey()
             return result.GetValue("masterKey").ToString();
         }
 
-        public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request) => _client.SendAsync(request);        
+        public async Task SetAppSetting(string name, string value)
+        {
+            var httpContent = new StringContent($"{{ {name} : {value} }}", Encoding.UTF8, "application/json");
+            await _client.PostAsync($"/api/settings", httpContent);
+        }
+
+        public Task<HttpResponseMessage> SendAsync(HttpRequestMessage request) => _client.SendAsync(request);
 
         protected virtual void Dispose(bool disposing)
         {

diff --git a/test/WebJobs.Script.Tests.E2E/Settings.cs → ...bJobs.Script.Tests.E2E.Shared/Settings.cs b/test/WebJobs.Script.Tests.E2E/Settings.cs → ...bJobs.Script.Tests.E2E.Shared/Settings.cs
@@ -1,11 +1,13 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using Microsoft.Extensions.Configuration;
 using System;
 using System.Configuration;
+using System.IO;
 using System.Text.RegularExpressions;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class Settings
     {
@@ -27,13 +29,21 @@ public class Settings
 
         public static string RuntimeExtensionPackageUrl => GetSettingValue(Constants.RuntimeExtensionPackageUrlSettingName);
 
+        public static string SiteMasterKey => GetSettingValue(Constants.TargetSiteMasterKey);
+
+        public static string SiteFunctionKey => GetSettingValue(Constants.TargetSiteFunctionKey);
+
+        public static string VM => GetSettingValue(Constants.VM);
+
         public static Uri SiteBaseAddress => new Uri($"https://{SiteName}.azurewebsites.net");
 
+        private static IConfiguration Config = null;
+
         public static string RuntimeVersion
         {
             get
             {
-                Match versionMatch = Regex.Match(RuntimeExtensionPackageUrl, "(?<version>\\d*\\.\\d*\\.\\d*)(-.*)?\\.zip$");
+                Match versionMatch = Regex.Match(RuntimeExtensionPackageUrl, "(\\.)(?<version>\\d*\\.\\d*\\.\\d*)(\\..*)?\\.zip$");
 
                 if (!versionMatch.Success)
                 {
@@ -48,14 +58,18 @@ public static string RuntimeVersion
 
         private static string GetSettingValue(string settingName)
         {
-            string value = Environment.GetEnvironmentVariable(settingName);
-
-            if (string.IsNullOrEmpty(value))
+            if (Config == null)
             {
-                value = ConfigurationManager.AppSettings.Get(settingName);
+                var builder = new ConfigurationBuilder().AddEnvironmentVariables();
+                if (File.Exists("local.settings.json"))
+                {
+                    builder.AddJsonFile("local.settings.json");
+                }
+                Config = builder.Build();
             }
 
-            return value;
+
+            return ConfigurationBinder.GetValue(Config, settingName, "default");
         }
     }
 }
diff --git a/...Jobs.Script.Tests.E2E/TelemetryContext.cs → ...ript.Tests.E2E.Shared/TelemetryContext.cs b/...Jobs.Script.Tests.E2E/TelemetryContext.cs → ...ript.Tests.E2E.Shared/TelemetryContext.cs
@@ -4,7 +4,7 @@
 using System;
 using Microsoft.ApplicationInsights;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public static class TelemetryContext
     {

diff --git a/...ebJobs.Script.Tests.E2E/TraceAttribute.cs → ...Script.Tests.E2E.Shared/TraceAttribute.cs b/...ebJobs.Script.Tests.E2E/TraceAttribute.cs → ...Script.Tests.E2E.Shared/TraceAttribute.cs
@@ -6,7 +6,7 @@
 using System.Reflection;
 using Xunit.Sdk;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class TestTraceAttribute : BeforeAfterTestAttribute
     {

diff --git a/...WebJobs.Script.Tests.E2E/TrackedAssert.cs → ....Script.Tests.E2E.Shared/TrackedAssert.cs b/...WebJobs.Script.Tests.E2E/TrackedAssert.cs → ....Script.Tests.E2E.Shared/TrackedAssert.cs
@@ -7,7 +7,7 @@
 using Microsoft.ApplicationInsights;
 using Xunit;
 
-namespace WebJobs.Script.EndToEndTests
+namespace WebJobs.Script.Tests.EndToEnd.Shared
 {
     public class TrackedAssert
     {

diff --git a/test/WebJobs.Script.Tests.E2E.Shared/WebJobs.Script.Tests.E2E.Shared.projitems b/test/WebJobs.Script.Tests.E2E.Shared/WebJobs.Script.Tests.E2E.Shared.projitems
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+    <HasSharedItems>true</HasSharedItems>
+    <SharedGUID>82d61060-8774-491c-9d73-9798a34c2399</SharedGUID>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration">
+    <Import_RootNamespace>WebJobs.Script.Tests.E2E.Shared</Import_RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildThisFileDirectory)ArmAuthenticationHelpers.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Constants.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Extensions.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Function.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)FunctionAppFixture.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)KuduClient.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Settings.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)TelemetryContext.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)TraceAttribute.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)TrackedAssert.cs" />
+  </ItemGroup>
+</Project>
diff --git a/test/WebJobs.Script.Tests.E2E.Shared/WebJobs.Script.Tests.E2E.Shared.shproj b/test/WebJobs.Script.Tests.E2E.Shared/WebJobs.Script.Tests.E2E.Shared.shproj
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>82d61060-8774-491c-9d73-9798a34c2399</ProjectGuid>
+    <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
+  </PropertyGroup>
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.Default.props" />
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.Common.props" />
+  <PropertyGroup />
+  <Import Project="WebJobs.Script.Tests.E2E.Shared.projitems" Label="Shared" />
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\CodeSharing\Microsoft.CodeSharing.CSharp.targets" />
+</Project>