Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce thrown exceptions #7527

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/Aspire.Hosting/Dcp/DcpExecutor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ internal sealed class DcpExecutor : IDcpExecutor, IAsyncDisposable
private readonly ResourceSnapshotBuilder _snapshotBuilder;

// Internal for testing.
internal ResiliencePipeline DeleteResourceRetryPipeline { get; set; }
internal ResiliencePipeline<bool> DeleteResourceRetryPipeline { get; set; }
internal ResiliencePipeline CreateServiceRetryPipeline { get; set; }
internal ResiliencePipeline WatchResourceRetryPipeline { get; set; }

Expand Down Expand Up @@ -1485,18 +1485,24 @@ async Task EnsureResourceDeletedAsync<T>(string resourceName) where T : CustomRe
// before resorting to more extreme measures.
if (!resourceNotFound)
{
await DeleteResourceRetryPipeline.ExecuteAsync(async (state, attemptCancellationToken) =>
var result = await DeleteResourceRetryPipeline.ExecuteAsync<bool, string>(async (state, attemptCancellationToken) =>
{
try
{
await _kubernetesService.GetAsync<T>(state, cancellationToken: attemptCancellationToken).ConfigureAwait(false);
throw new DistributedApplicationException($"Failed to delete '{state}' successfully before restart.");
return false;
}
catch (HttpOperationException ex) when (ex.Response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
// Success.
return true;
}
}, resourceName, cancellationToken).ConfigureAwait(false);

if (!result)
{
throw new DistributedApplicationException($"Failed to delete '{resourceName}' successfully before restart.");
}
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/Aspire.Hosting/Dcp/DcpPipelineBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,24 @@ namespace Aspire.Hosting.Dcp;

internal static class DcpPipelineBuilder
{
public static ResiliencePipeline BuildDeleteRetryPipeline(ILogger logger)
public static ResiliencePipeline<bool> BuildDeleteRetryPipeline(ILogger logger)
{
var ensureDeleteRetryStrategy = new RetryStrategyOptions()
var ensureDeleteRetryStrategy = new RetryStrategyOptions<bool>()
{
BackoffType = DelayBackoffType.Exponential,
Delay = TimeSpan.FromMilliseconds(200),
UseJitter = true,
MaxRetryAttempts = 10, // Cumulative time for all attempts amounts to about 15 seconds
MaxDelay = TimeSpan.FromSeconds(3),
ShouldHandle = new PredicateBuilder().Handle<Exception>(),
ShouldHandle = args => ValueTask.FromResult(!args.Outcome.Result),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about when an exception occurs that isn't Http NotFound? Should we retry on that?

OnRetry = (retry) =>
{
logger.LogDebug("Retrying check for deleted resource. Attempt: {Attempt}. Error message: {ErrorMessage}", retry.AttemptNumber, retry.Outcome.Exception?.Message);
return ValueTask.CompletedTask;
}
};

var execution = new ResiliencePipelineBuilder().AddRetry(ensureDeleteRetryStrategy).Build();
var execution = new ResiliencePipelineBuilder<bool>().AddRetry(ensureDeleteRetryStrategy).Build();
return execution;
}

Expand Down
24 changes: 7 additions & 17 deletions src/Aspire.Hosting/Health/ResourceHealthCheckService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,6 @@ internal class ResourceMonitorState
private readonly object _lock = new object();
private readonly string _resourceName;
private TaskCompletionSource? _delayInterruptTcs;
private CancellationTokenSource? _delayCts;

public ResourceMonitorState(ILogger logger, ResourceEvent initialEvent, CancellationToken serviceStoppingToken)
{
Expand Down Expand Up @@ -334,6 +333,7 @@ public void SetLatestEvent(ResourceEvent resourceEvent)

internal async Task<bool> DelayAsync(ResourceEvent? currentEvent, TimeSpan delay, CancellationToken cancellationToken)
{
Task delayInterruptedTask;
lock (_lock)
{
// The event might have changed before delay was called. Interrupt immediately if required.
Expand All @@ -342,26 +342,16 @@ internal async Task<bool> DelayAsync(ResourceEvent? currentEvent, TimeSpan delay
_logger.LogTrace("Health monitoring delay interrupted for resource '{Resource}'.", _resourceName);
return true;
}
if (_delayCts == null || !_delayCts.TryReset())
{
_delayCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
}
_delayInterruptTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
delayInterruptedTask = _delayInterruptTcs.Task;
}

var completedTask = await Task.WhenAny(Task.Delay(delay, _delayCts.Token), _delayInterruptTcs.Task).ConfigureAwait(false);
// Don't throw to avoid writing the thrown exception to the debug console.
// See https://github.com/dotnet/aspire/issues/7486
danmoseley marked this conversation as resolved.
Show resolved Hide resolved
await delayInterruptedTask.WaitAsync(delay, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
var delayInterrupted = delayInterruptedTask.IsCompletedSuccessfully == true;

if (completedTask != _delayInterruptTcs.Task)
{
// Task.Delay won.
return false;
}
else
{
// Delay was interrupted. Cancel the delay task so it doesn't hang around when not needed.
_delayCts.Cancel();
return true;
}
return delayInterrupted;
}

private static bool ShouldInterrupt(ResourceEvent currentEvent, ResourceEvent previousEvent)
Expand Down
2 changes: 1 addition & 1 deletion tests/Aspire.Hosting.Tests/Dcp/DcpExecutorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1106,7 +1106,7 @@ public async Task ErrorIfResourceNotDeletedBeforeRestart()
var appExecutor = CreateAppExecutor(distributedAppModel, kubernetesService: kubernetesService, events: dcpEvents);

// Set a custom pipeline without retries or delays to avoid waiting.
appExecutor.DeleteResourceRetryPipeline = new ResiliencePipelineBuilder().Build();
appExecutor.DeleteResourceRetryPipeline = new ResiliencePipelineBuilder<bool>().Build();

await appExecutor.RunApplicationAsync();

Expand Down