Reenable http throttles (Azure#2945)

.gitignore

@@ -14,6 +14,7 @@ Publish
 msbuild.log
 package-lock.json
 **/Properties/launchSettings.json
+**/project.assets.json
 
 /packages
 tools/ExtensionsMetadataGenerator/packages

src/WebJobs.Script.WebHost/Configuration/JobHostOptionsSetup.cs

@@ -23,6 +23,7 @@ public JobHostOptionsSetup(IConfiguration configuration)
 
         public void Configure(JobHostOptions options)
         {
+            // TODO: Why isn't this code doing anything?
             IConfigurationSection jobHostSection = _configuration.GetSection(ConfigurationSectionNames.JobHost);
         }
     }

src/WebJobs.Script.WebHost/HttpRequestQueue.cs

@@ -0,0 +1,102 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Azure.WebJobs.Extensions.Http;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost
+{
+    /// <summary>
+    /// Encapsulates an http request queue used for request throttling. See <see cref="HttpThrottleMiddleware"/>.
+    /// This has been factored as its own service to ensure that it's lifetime stays tied to host the host
+    /// instance lifetime, not the middleware lifetime which is longer lived.
+    /// </summary>
+    internal class HttpRequestQueue
+    {
+        private readonly IOptions<HttpOptions> _httpOptions;
+        private ActionBlock<HttpRequestItem> _requestQueue;
+
+        public HttpRequestQueue(IOptions<HttpOptions> httpOptions)
+        {
+            _httpOptions = httpOptions;
+
+            if (_httpOptions.Value.MaxOutstandingRequests != DataflowBlockOptions.Unbounded ||
+                _httpOptions.Value.MaxConcurrentRequests != DataflowBlockOptions.Unbounded)
+            {
+                InitializeRequestQueue();
+            }
+        }
+
+        /// <summary>
+        /// Gets a value indicating whether request queueing is enabled.
+        /// </summary>
+        public bool Enabled => _requestQueue != null;
+
+        public async Task<bool> Post(HttpContext httpContext, RequestDelegate next)
+        {
+            // enqueue the request workitem
+            var item = new HttpRequestItem
+            {
+                HttpContext = httpContext,
+                Next = next,
+                CompletionSource = new TaskCompletionSource<object>()
+            };
+
+            if (_requestQueue.Post(item))
+            {
+                await item.CompletionSource.Task;
+                return true;
+            }
+            else
+            {
+                // no more requests can be queued at this time
+                return false;
+            }
+        }
+
+        private void InitializeRequestQueue()
+        {
+            // if throttles are enabled, initialize the queue
+            var blockOptions = new ExecutionDataflowBlockOptions
+            {
+                MaxDegreeOfParallelism = _httpOptions.Value.MaxConcurrentRequests,
+                BoundedCapacity = _httpOptions.Value.MaxOutstandingRequests
+            };
+
+            _requestQueue = new ActionBlock<HttpRequestItem>(async item =>
+            {
+                try
+                {
+                    await item.Next.Invoke(item.HttpContext);
+                    item.CompletionSource.SetResult(null);
+                }
+                catch (Exception ex)
+                {
+                    item.CompletionSource.SetException(ex);
+                }
+            }, blockOptions);
+        }
+
+        private class HttpRequestItem
+        {
+            /// <summary>
+            /// Gets or sets the request context to process.
+            /// </summary>
+            public HttpContext HttpContext { get; set; }
+
+            /// <summary>
+            /// Gets or sets the completion delegate for the request.
+            /// </summary>
+            public RequestDelegate Next { get; set; }
+
+            /// <summary>
+            /// Gets or sets the completion source to use.
+            /// </summary>
+            public TaskCompletionSource<object> CompletionSource { get; set; }
+        }
+    }
+}

src/WebJobs.Script.WebHost/Middleware/HttpThrottleMiddleware.cs

@@ -0,0 +1,78 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.ObjectModel;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Azure.WebJobs.Extensions.Http;
+using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Scale;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost.Middleware
+{
+    internal class HttpThrottleMiddleware
+    {
+        private readonly RequestDelegate _next;
+        private readonly TimeSpan? _performanceCheckInterval;
+        private readonly ILogger _logger;
+        private DateTime _lastPerformanceCheck;
+        private bool _rejectRequests;
+
+        public HttpThrottleMiddleware(RequestDelegate next, ILoggerFactory loggerFactory, TimeSpan? performanceCheckInterval = null)
+        {
+            _next = next;
+            _performanceCheckInterval = performanceCheckInterval ?? TimeSpan.FromSeconds(15);
+            _logger = loggerFactory?.CreateLogger("Host.Extensions.Http.HttpThrottleMiddleware");
+        }
+
+        public async Task Invoke(HttpContext httpContext, IOptions<HttpOptions> httpOptions, HttpRequestQueue requestQueue, HostPerformanceManager performanceManager, IMetricsLogger metricsLogger)
+        {
+            if (httpOptions.Value.DynamicThrottlesEnabled &&
+               ((DateTime.UtcNow - _lastPerformanceCheck) > _performanceCheckInterval))
+            {
+                // only check host status periodically
+                Collection<string> exceededCounters = new Collection<string>();
+                _rejectRequests = performanceManager.IsUnderHighLoad(exceededCounters);
+                _lastPerformanceCheck = DateTime.UtcNow;
+                if (_rejectRequests)
+                {
+                    _logger.LogWarning($"Thresholds for the following counters have been exceeded: [{string.Join(", ", exceededCounters)}]");
+                }
+            }
+
+            if (_rejectRequests)
+            {
+                // we're currently in reject mode, so reject the request and
+                // call the next delegate without calling base
+                RejectRequest(httpContext, metricsLogger);
+                return;
+            }
+
+            if (requestQueue.Enabled)
+            {
+                var success = await requestQueue.Post(httpContext, _next);
+                if (!success)
+                {
+                    _logger?.LogInformation($"Http request queue limit of {httpOptions.Value.MaxOutstandingRequests} has been exceeded.");
+                    RejectRequest(httpContext, metricsLogger);
+                }
+            }
+            else
+            {
+                // queue is not enabled, so just dispatch the request directly
+                await _next.Invoke(httpContext);
+            }
+        }
+
+        private void RejectRequest(HttpContext httpContext, IMetricsLogger metricsLogger)
+        {
+            metricsLogger.LogEvent(MetricEventNames.FunctionInvokeThrottled);
+
+            httpContext.Response.StatusCode = 429;
+            httpContext.Response.Headers.Add(ScriptConstants.AntaresScaleOutHeaderName, "1");
+        }
+    }
+}

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

@@ -49,8 +49,8 @@
     <PackageReference Include="Microsoft.AspNetCore.Buffering" Version="0.4.0-preview2-28189" />
     <PackageReference Include="Microsoft.Azure.AppService.Proxy.Client" Version="2.0.5350001-beta-fc119b98" />
     <PackageReference Include="Microsoft.AspNetCore.Server.IIS" Version="2.1.0-a-oob-2-1-oob-17035" />
-    <PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="3.0.0-beta7-11417" />
-    <PackageReference Include="Microsoft.Azure.WebJobs.Logging" Version="3.0.0-beta7-11417" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="3.0.0-beta7-11418" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Logging" Version="3.0.0-beta7-11418" />
     <PackageReference Include="Microsoft.Azure.WebSites.DataProtection" Version="2.1.88-alpha" />
     <PackageReference Include="System.Net.Primitives" Version="4.3.0" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.1" />

src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs

@@ -39,6 +39,7 @@ public static IApplicationBuilder UseWebJobsScriptHost(this IApplicationBuilder
             builder.UseMiddleware<HttpExceptionMiddleware>();
             builder.UseMiddleware<ResponseBufferingMiddleware>();
             builder.UseMiddleware<HomepageMiddleware>();
+            builder.UseMiddleware<HttpThrottleMiddleware>();
             builder.UseMiddleware<FunctionInvocationMiddleware>();
             builder.UseMiddleware<HostWarmupMiddleware>();
 

src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs

@@ -46,6 +46,7 @@ public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceP
                 })
                 .ConfigureServices(services =>
                 {
+                    services.AddSingleton<HttpRequestQueue>();
                     services.AddSingleton<IHostLifetime, JobHostHostLifetime>();
                     services.TryAddSingleton<IWebJobsExceptionHandler, WebScriptHostExceptionHandler>();
                     services.AddSingleton<IScriptJobHostEnvironment, WebScriptJobHostEnvironment>();

src/WebJobs.Script/Config/ConfigurationSectionNames.cs

@@ -13,5 +13,6 @@ public static class ConfigurationSectionNames
         public const string WebHost = "AzureFunctionsWebHost";
         public const string JobHost = "AzureFunctionsJobHost";
         public const string JobHostLogger = "logger";
+        public const string HealthMonitor = "healthMonitor";
     }
 }

src/WebJobs.Script/Config/HostHealthMonitorConfiguration.cs → src/WebJobs.Script/Config/HostHealthMonitorOptions.cs

@@ -5,11 +5,11 @@
 
 namespace Microsoft.Azure.WebJobs.Script
 {
-    public class HostHealthMonitorConfiguration
+    public class HostHealthMonitorOptions
     {
         internal const float DefaultCounterThreshold = 0.80F;
 
-        public HostHealthMonitorConfiguration()
+        public HostHealthMonitorOptions()
         {
             Enabled = true;
 

src/WebJobs.Script/Config/HostHealthMonitorOptionsSetup.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using Microsoft.Azure.WebJobs.Script.Configuration;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Script.Config
+{
+    internal class HostHealthMonitorOptionsSetup : IConfigureOptions<HostHealthMonitorOptions>
+    {
+        private readonly IConfiguration _configuration;
+
+        public HostHealthMonitorOptionsSetup(IConfiguration configuration)
+        {
+            _configuration = configuration;
+        }
+
+        public void Configure(HostHealthMonitorOptions options)
+        {
+            var section = _configuration.GetSection(ConfigurationSectionNames.HealthMonitor);
+            section.Bind(options);
+        }
+    }
+}

src/WebJobs.Script/Config/ScriptJobHostOptions.cs

@@ -18,7 +18,6 @@ public ScriptJobHostOptions()
         {
             FileWatchingEnabled = true;
             FileLoggingMode = FileLoggingMode.Never;
-            HostHealthMonitor = new HostHealthMonitorConfiguration();
             InstanceId = Guid.NewGuid().ToString();
             WatchDirectories = new Collection<string>();
         }
@@ -128,10 +127,5 @@ public ImmutableArray<string> RootScriptDirectorySnapshot
         /// locally or via CLI.
         /// </summary>
         public bool IsSelfHost { get; set; }
-
-        /// <summary>
-        /// Gets the <see cref="HostHealthMonitorConfiguration"/> to use.
-        /// </summary>
-        public HostHealthMonitorConfiguration HostHealthMonitor { get; }
     }
 }