From 16f1755a808ed59adaf7c7707e29d69f6ef3ff61 Mon Sep 17 00:00:00 2001 From: Lana Parezanin Date: Wed, 20 Nov 2024 14:34:57 -0800 Subject: [PATCH 01/44] Fixing gallery build error --- .../BlobStorageVulnerabilityWriterFacts.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs index 12791043e2..0c4ec02aa6 100644 --- a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs +++ b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs @@ -165,6 +165,11 @@ public override Task> ListAsync(bool getMetadata, C return _storage.ListAsync(getMetadata, cancellationToken); } + public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + return _storage.ListAsync(getMetadata, cancellationToken); + } + public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { return _storage.SetMetadataAsync(resourceUri, metadata); From 9155726fb0cac8b3fa62046c80a5c77ea53e6112 Mon Sep 17 00:00:00 2001 From: Lana Parezanin Date: Wed, 20 Nov 2024 14:15:15 -0800 Subject: [PATCH 02/44] Testfix --- .../DetailedReportPostProcessorFacts.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs index 67d77e7f38..8acd723d96 100644 --- a/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs +++ b/tests/Stats.PostProcessReports.Tests/DetailedReportPostProcessorFacts.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -33,13 +33,13 @@ public class DetailedReportPostProcessorFacts public async Task DoesntStartIfNoSuccessFile() { _sourceStorageMock - .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .Setup(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new List()); await _target.CopyReportsAsync(); _sourceStorageMock - .Verify(ss => ss.ListAsync(It.IsAny(), It.IsAny()), Times.Once); + .Verify(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny()), Times.Once); _sourceStorageMock.VerifyNoOtherCalls(); _workStorageMock.VerifyNoOtherCalls(); _destinationStorageMock.VerifyNoOtherCalls(); @@ -186,7 +186,7 @@ public async Task SkipsProcessedFiles() { "FilesCreated", "123" } }; _workStorageMock - .Setup(ss => ss.ListAsync(It.IsAny(), It.IsAny())) + .Setup(ss => ss.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(() => new List(_workFiles.Select(f => Blob( _workStorageMock, f, @@ -286,7 +286,7 @@ private static void SetupStorageMock(Mock mock, string baseUrl, Func s.ListAsync(It.IsAny(), It.IsAny())) + .Setup(s => s.ListTopLevelAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(() => new List(files().Select(f => Blob(mock, f)))); mock .Setup(s => s.ResolveUri(It.IsAny())) From fabc79fc7085adc919fd3a5560b704858e29810c Mon Sep 17 00:00:00 2001 From: Lana Parezanin Date: Mon, 28 Oct 2024 11:46:48 -0700 Subject: [PATCH 03/44] Bugfix --- src/NuGet.Services.Storage/AzureStorage.cs | 24 ++++++++++++++++++- .../DetailedReportPostProcessor.cs | 12 +++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/NuGet.Services.Storage/AzureStorage.cs b/src/NuGet.Services.Storage/AzureStorage.cs index 8e54874625..1c30798c24 100644 --- a/src/NuGet.Services.Storage/AzureStorage.cs +++ b/src/NuGet.Services.Storage/AzureStorage.cs @@ -189,7 +189,29 @@ public override async Task> ListAsync(bool getMetad await foreach (BlobHierarchyItem blob in _directory.GetBlobsByHierarchyAsync(traits: blobTraits, prefix: _path)) { - blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + } + + return blobList; + } + + public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + var prefix = _path.Trim('/') + '/'; + var blobTraits = new BlobTraits(); + if (getMetadata) + { + blobTraits |= BlobTraits.Metadata; + } + + var blobList = new List(); + + await foreach (BlobHierarchyItem blob in _directory.GetBlobsByHierarchyAsync(traits: blobTraits, prefix: prefix, delimiter: "/")) + { + if (!blob.IsPrefix) + { + blobList.Add(await GetStorageListItemAsync(_directory.GetBlockBlobClient(blob.Blob.Name))); + } } return blobList; diff --git a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs index 8441fa4d2e..4ab9cb0a3b 100644 --- a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs +++ b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs @@ -120,7 +120,7 @@ private async Task ProcessBlobs(List jsonBlobs, CancellationTok foreach (var sourceBlob in jsonBlobs) { var blobName = GetBlobName(sourceBlob); - var workBlobUri = _workStorage.ResolveUri(blobName); + var workBlobUri = _workStorage.ResolveUri(_configuration.WorkPath + blobName); var sourceBlobStats = new BlobStatistics(); var individualReports = await ProcessSourceBlobAsync(sourceBlob, sourceBlobStats, totals); using (_logger.BeginScope("Processing {BlobName}", blobName)) @@ -154,7 +154,7 @@ private async Task ProcessBlobs(List jsonBlobs, CancellationTok } } } - var jobSucceededUrl = _workStorage.ResolveUri(JobSucceededFilename); + var jobSucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + JobSucceededFilename); var jobSucceededContent = new StringStorageContent("", TextContentType); await _workStorage.Save(jobSucceededUrl, jobSucceededContent, overwrite: true, cancellationToken: cancellationToken); _telemetryService.ReportTotals(totals.SourceFilesProcessed, totals.TotalLinesProcessed, totals.TotalFilesCreated, totals.TotalLinesFailed); @@ -202,13 +202,13 @@ private async Task CopySourceBlobsAsync(List jsonBlobs, Cancell foreach (var sourceBlob in jsonBlobs) { var blobName = GetBlobName(sourceBlob); - var targetUrl = _workStorage.ResolveUri(blobName); + var targetUrl = _workStorage.ResolveUri(_configuration.WorkPath + blobName); _logger.LogInformation("{SourceBlobUri} ({BlobName})", sourceBlob.Uri.AbsoluteUri, blobName); _logger.LogInformation("{WorkBlobUrl}", targetUrl); await _sourceStorage.CopyAsync(sourceBlob.Uri, _workStorage, targetUrl, destinationProperties: null, cancellationToken); } var copySucceededContent = new StringStorageContent("", TextContentType); - var copySucceededUrl = _workStorage.ResolveUri(CopySucceededFilename); + var copySucceededUrl = _workStorage.ResolveUri(_configuration.WorkPath + CopySucceededFilename); await _workStorage.Save(copySucceededUrl, copySucceededContent, overwrite: true, cancellationToken: cancellationToken); } @@ -245,7 +245,7 @@ private async Task> ProcessSourceBlobAsync( var sw = Stopwatch.StartNew(); var numLines = 0; var individualReports = new ConcurrentBag(); - var workStorageUrl = _workStorage.ResolveUri(GetBlobName(sourceBlob)); + var workStorageUrl = _workStorage.ResolveUri(_configuration.WorkPath + GetBlobName(sourceBlob)); var storageContent = await _workStorage.Load(workStorageUrl, CancellationToken.None); using (var sourceStream = storageContent.GetContentStream()) using (var streamReader = new StreamReader(sourceStream)) @@ -351,7 +351,7 @@ private async Task WriteReports( continue; } var outFilename = $"recentpopularitydetail_{data.PackageId.ToLowerInvariant()}.json"; - var destinationUri = _destinationStorage.ResolveUri(outFilename); + var destinationUri = _destinationStorage.ResolveUri(_configuration.DestinationPath + outFilename); var storageContent = new StringStorageContent(details.Data, JsonContentType); await _destinationStorage.Save(destinationUri, storageContent, overwrite: true, cancellationToken: cancellationToken); From f8c582992f52e6039eb4af771c32c133645181bd Mon Sep 17 00:00:00 2001 From: Lana Parezanin Date: Mon, 21 Oct 2024 17:22:34 -0700 Subject: [PATCH 04/44] Added TopLevel --- global.json | 4 ++-- src/NuGet.Services.Storage/AggregateStorage.cs | 7 +++++-- src/NuGet.Services.Storage/FileStorage.cs | 5 ++++- src/NuGet.Services.Storage/IStorage.cs | 4 +++- src/NuGet.Services.Storage/Storage.cs | 7 +++++++ .../DetailedReportPostProcessor.cs | 9 +++++---- 6 files changed, 26 insertions(+), 10 deletions(-) diff --git a/global.json b/global.json index 32da22419b..4c1d7e5aaa 100644 --- a/global.json +++ b/global.json @@ -1,7 +1,7 @@ { "sdk": { - "version": "8.0.303", + "version": "6.0.0", "rollForward": "latestFeature", "allowPrerelease": false } -} \ No newline at end of file +} diff --git a/src/NuGet.Services.Storage/AggregateStorage.cs b/src/NuGet.Services.Storage/AggregateStorage.cs index 4fc61ab402..788153f939 100644 --- a/src/NuGet.Services.Storage/AggregateStorage.cs +++ b/src/NuGet.Services.Storage/AggregateStorage.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -111,9 +111,12 @@ public override async Task> ListAsync(bool getMetad return await _primaryStorage.ListAsync(getMetadata, cancellationToken); } + public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => + await ListAsync(getMetadata, cancellationToken); + public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { throw new NotImplementedException(); } } -} \ No newline at end of file +} diff --git a/src/NuGet.Services.Storage/FileStorage.cs b/src/NuGet.Services.Storage/FileStorage.cs index 32732ab85f..f6f815dfe8 100644 --- a/src/NuGet.Services.Storage/FileStorage.cs +++ b/src/NuGet.Services.Storage/FileStorage.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -57,6 +57,9 @@ public override Task> ListAsync(bool getMetadata, C return Task.FromResult>(List(getMetadata)); } + public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => + ListAsync(getMetadata, cancellationToken); + public string Path { get; diff --git a/src/NuGet.Services.Storage/IStorage.cs b/src/NuGet.Services.Storage/IStorage.cs index ccd11b146e..cb8dd46fc2 100644 --- a/src/NuGet.Services.Storage/IStorage.cs +++ b/src/NuGet.Services.Storage/IStorage.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -20,6 +20,8 @@ public interface IStorage Uri ResolveUri(string relativeUri); IEnumerable List(bool getMetadata); Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); + Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); + Task CopyAsync( Uri sourceUri, IStorage destinationStorage, diff --git a/src/NuGet.Services.Storage/Storage.cs b/src/NuGet.Services.Storage/Storage.cs index 2f428d0501..ae66aeb4fe 100644 --- a/src/NuGet.Services.Storage/Storage.cs +++ b/src/NuGet.Services.Storage/Storage.cs @@ -156,6 +156,7 @@ public async Task LoadString(Uri resourceUri, CancellationToken cancella public abstract Task ExistsAsync(string fileName, CancellationToken cancellationToken); public abstract IEnumerable List(bool getMetadata); public abstract Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); + public abstract Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); public bool Verbose { @@ -209,6 +210,12 @@ protected string GetName(Uri uri) { name = name.Substring(0, name.IndexOf("#")); } + + if (name.Contains("?")) + { + name = name.Substring(0, name.IndexOf("?")); + } + return name; } diff --git a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs index 4ab9cb0a3b..13a720ad4a 100644 --- a/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs +++ b/src/Stats.PostProcessReports/DetailedReportPostProcessor.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -10,6 +10,7 @@ using System.Net; using System.Threading; using System.Threading.Tasks; +using Azure.Core; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; @@ -214,14 +215,14 @@ private async Task CopySourceBlobsAsync(List jsonBlobs, Cancell private async Task> EnumerateSourceBlobsAsync() { - var blobs = await _sourceStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _sourceStorage.ListTopLevelAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } private async Task> EnumerateWorkStorageBlobsAsync() { - var blobs = await _workStorage.ListAsync(getMetadata: true, cancellationToken: CancellationToken.None); + var blobs = await _workStorage.ListTopLevelAsync(getMetadata: true, cancellationToken: CancellationToken.None); return blobs.ToList(); } @@ -311,7 +312,7 @@ private static bool BlobMetadataExists(StorageListItem sourceBlob, TotalStats to private static string GetBlobName(StorageListItem blob) { - var path = blob.Uri.AbsoluteUri; + var path = blob.Uri.GetComponents(UriComponents.Path, UriFormat.UriEscaped); var lastSlash = path.LastIndexOf('/'); if (lastSlash < 0) { From 4a4b0f16f36a5d5c23e6aa4928c7a411692dc501 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 25 Nov 2024 13:41:39 -0800 Subject: [PATCH 05/44] Has right .net sdk version now --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 4c1d7e5aaa..ce2e41208c 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "6.0.0", + "version": "8.0.303", "rollForward": "latestFeature", "allowPrerelease": false } From b14c3fce7215154470fcc366f5b264d7227e0508 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 25 Nov 2024 13:42:55 -0800 Subject: [PATCH 06/44] Changed toplevel error --- src/NuGet.Services.Storage/AggregateStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGet.Services.Storage/AggregateStorage.cs b/src/NuGet.Services.Storage/AggregateStorage.cs index 788153f939..e7bf06e657 100644 --- a/src/NuGet.Services.Storage/AggregateStorage.cs +++ b/src/NuGet.Services.Storage/AggregateStorage.cs @@ -112,7 +112,7 @@ public override async Task> ListAsync(bool getMetad } public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => - await ListAsync(getMetadata, cancellationToken); + await ListTopLevelAsync(getMetadata, cancellationToken); public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { From 97e310d1776f1756ef4b042dfd079321149da1f3 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 25 Nov 2024 13:44:50 -0800 Subject: [PATCH 07/44] Changed toplevel error pt 2 --- src/NuGet.Services.Storage/FileStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGet.Services.Storage/FileStorage.cs b/src/NuGet.Services.Storage/FileStorage.cs index f6f815dfe8..73dce3a41b 100644 --- a/src/NuGet.Services.Storage/FileStorage.cs +++ b/src/NuGet.Services.Storage/FileStorage.cs @@ -58,7 +58,7 @@ public override Task> ListAsync(bool getMetadata, C } public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => - ListAsync(getMetadata, cancellationToken); + ListTopLevelAsync(getMetadata, cancellationToken); public string Path { From 3d55f8306bcb666c67697bd50e1c0c25a7e65a79 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 25 Nov 2024 13:46:09 -0800 Subject: [PATCH 08/44] Changed toplevel error pt 3 --- .../BlobStorageVulnerabilityWriterFacts.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs index 0c4ec02aa6..f45a184924 100644 --- a/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs +++ b/tests/GitHubVulnerabilities2v3.Facts/BlobStorageVulnerabilityWriterFacts.cs @@ -167,7 +167,7 @@ public override Task> ListAsync(bool getMetadata, C public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) { - return _storage.ListAsync(getMetadata, cancellationToken); + return _storage.ListTopLevelAsync(getMetadata, cancellationToken); } public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) From cbf5066a840dbfacbe3f2a288751a9685b2c91d7 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 13 Dec 2024 17:28:05 -0800 Subject: [PATCH 09/44] Bugfix --- src/NuGet.Services.Storage/AggregateStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGet.Services.Storage/AggregateStorage.cs b/src/NuGet.Services.Storage/AggregateStorage.cs index e7bf06e657..3bbf0f06dd 100644 --- a/src/NuGet.Services.Storage/AggregateStorage.cs +++ b/src/NuGet.Services.Storage/AggregateStorage.cs @@ -112,7 +112,7 @@ public override async Task> ListAsync(bool getMetad } public override async Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => - await ListTopLevelAsync(getMetadata, cancellationToken); + await _primaryStorage.ListTopLevelAsync(getMetadata, cancellationToken); public override Task SetMetadataAsync(Uri resourceUri, IDictionary metadata) { From bb807d0492e62c1f76c0293007da5b83310a6fc7 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 13 Dec 2024 17:51:06 -0800 Subject: [PATCH 10/44] Bugfix2 --- src/NuGet.Services.Storage/FileStorage.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/NuGet.Services.Storage/FileStorage.cs b/src/NuGet.Services.Storage/FileStorage.cs index 73dce3a41b..ea724df432 100644 --- a/src/NuGet.Services.Storage/FileStorage.cs +++ b/src/NuGet.Services.Storage/FileStorage.cs @@ -57,8 +57,10 @@ public override Task> ListAsync(bool getMetadata, C return Task.FromResult>(List(getMetadata)); } - public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) => - ListTopLevelAsync(getMetadata, cancellationToken); + public override Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } public string Path { From 03e111cae88884e8cc2091cb1f69058c546c263d Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 13 Dec 2024 18:09:10 -0800 Subject: [PATCH 11/44] Added a comment --- src/NuGet.Services.Storage/IStorage.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/NuGet.Services.Storage/IStorage.cs b/src/NuGet.Services.Storage/IStorage.cs index cb8dd46fc2..f385939008 100644 --- a/src/NuGet.Services.Storage/IStorage.cs +++ b/src/NuGet.Services.Storage/IStorage.cs @@ -20,6 +20,12 @@ public interface IStorage Uri ResolveUri(string relativeUri); IEnumerable List(bool getMetadata); Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); + + /// + /// Lists the top-level items in the storage. + /// Unlike , this method uses a delimiter to comply with new SDK requirements. + /// Without the trailing slash, the method would return an extra entry for the path itself. + /// Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); Task CopyAsync( From 7693ca65b97ccda1b8fd94a9e1518276fda5626b Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 16 Dec 2024 11:26:26 -0800 Subject: [PATCH 12/44] Clarified comment --- src/NuGet.Services.Storage/IStorage.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/NuGet.Services.Storage/IStorage.cs b/src/NuGet.Services.Storage/IStorage.cs index f385939008..6e00bceb2e 100644 --- a/src/NuGet.Services.Storage/IStorage.cs +++ b/src/NuGet.Services.Storage/IStorage.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -19,13 +20,11 @@ public interface IStorage Uri BaseAddress { get; } Uri ResolveUri(string relativeUri); IEnumerable List(bool getMetadata); + + //Lists all children of the storage(including the ones contained in subdirectories). Task> ListAsync(bool getMetadata, CancellationToken cancellationToken); - /// - /// Lists the top-level items in the storage. - /// Unlike , this method uses a delimiter to comply with new SDK requirements. - /// Without the trailing slash, the method would return an extra entry for the path itself. - /// + //Lists immediate children of the storage assuming directory-like structure Task> ListTopLevelAsync(bool getMetadata, CancellationToken cancellationToken); Task CopyAsync( From e9a646c99fab7ff218558ae15cb65156a31d9ad1 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 27 Dec 2024 11:26:41 -0800 Subject: [PATCH 13/44] AzureStatsLogDestination is now using new SDK --- .../Collect/AzureStatsLogDestination.cs | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs index be77f90b6e..cbcd91ecd7 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs @@ -1,4 +1,4 @@ - + // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. @@ -7,9 +7,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; using ICSharpCode.SharpZipLib.GZip; +using Azure.Storage.Blobs.Models; namespace Stats.AzureCdnLogs.Common.Collect { @@ -21,17 +21,15 @@ public class AzureStatsLogDestination : ILogDestination private const string ContentTypeGzip = "application/x-gzip"; private const string ContentTypeText = "text/plain"; - private CloudStorageAccount _azureAccount; - private CloudBlobClient _cloudBlobClient; - private CloudBlobContainer _cloudBlobContainer; + private BlobServiceClient _blobServiceClient; + private BlobContainerClient _blobContainerClient; private readonly ILogger _logger; - public AzureStatsLogDestination(CloudStorageAccount storageAccount, string containerName, ILogger logger) + public AzureStatsLogDestination(BlobServiceClient blobServiceClient, string containerName, ILogger logger) { - _azureAccount = storageAccount; - _cloudBlobClient = _azureAccount.CreateCloudBlobClient(); - _cloudBlobContainer = _cloudBlobClient.GetContainerReference(containerName); - _cloudBlobContainer.CreateIfNotExists(); + _blobServiceClient = blobServiceClient; + _blobContainerClient = _blobServiceClient.GetBlobContainerClient(containerName); + _blobContainerClient.CreateIfNotExists(); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -51,7 +49,7 @@ public AzureStatsLogDestination(CloudStorageAccount storageAccount, string conta public async Task TryWriteAsync(Stream inputStream, Action writeAction, string destinationFileName, ContentType destinationContentType, CancellationToken token) { _logger.LogInformation("WriteAsync: Start to write to {DestinationFileName}. ContentType is {ContentType}.", - $"{_cloudBlobContainer.StorageUri}{_cloudBlobContainer.Name}{destinationFileName}", + $"{_blobContainerClient.Uri}{destinationFileName}", destinationContentType); if (token.IsCancellationRequested) { @@ -60,16 +58,19 @@ public async Task TryWriteAsync(Stream inputStream, Action } try { - var blob = _cloudBlobContainer.GetBlockBlobReference(destinationFileName); - blob.Properties.ContentType = GetContentType(destinationContentType); + var blobClient = _blobContainerClient.GetBlobClient(destinationFileName); + var options = new BlobOpenWriteOptions() + { + HttpHeaders = new BlobHttpHeaders { ContentType = GetContentType(destinationContentType) } + }; // If the blob was written already to the destination do not do anything. // This should not happen if the renew task was correctly scheduled. Add the check just in case that the renew task was not scheduled in time and a different process already processed the file. - if (!(await blob.ExistsAsync())) + if (!(await blobClient.ExistsAsync(token))) { // Do not use using to not automatically commit on dispose // https://github.com/Azure/azure-storage-net/issues/832 - var resultStream = await blob.OpenWriteAsync(); + var resultStream = await blobClient.OpenWriteAsync(overwrite: true, options, cancellationToken: token); if (destinationContentType == ContentType.GZip) { using (var resultGzipStream = new GZipOutputStream(resultStream)) @@ -83,12 +84,10 @@ public async Task TryWriteAsync(Stream inputStream, Action { writeAction(inputStream, resultStream); } - if (!(await blob.ExistsAsync())) - { - resultStream.Commit(); - _logger.LogInformation("WriteAsync: End write to {DestinationFileName}", destinationFileName); - return new AsyncOperationResult(true, null); - } + + await resultStream.FlushAsync(); + _logger.LogInformation("WriteAsync: End write to {DestinationFileName}", destinationFileName); + return new AsyncOperationResult(true, null); } _logger.LogInformation("WriteAsync: The destination file {DestinationFileName}, was already present.", destinationFileName); return new AsyncOperationResult(false, null); From 901b38d4d2b3c0c8fdb49a10201afa4735308410 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Tue, 14 Jan 2025 09:44:48 -0800 Subject: [PATCH 14/44] Temp changes --- .../AzureHelpers/AzureBlobLeaseManager.cs | 30 ++++++++++++++--- .../AzureHelpers/AzureBlobLockResult.cs | 4 +-- .../Collect/AzureStatsLogSource.cs | 32 ++++++++++--------- .../Stats.AzureCdnLogs.Common.csproj | 3 +- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 23 +++++++------ 5 files changed, 60 insertions(+), 32 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 33832b85a7..bc66d666d2 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -1,12 +1,15 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; +using Stats.AzureCdnLogs.Common.Collect; +using NuGet.Services.Storage; +//using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage.Blob; namespace Stats.AzureCdnLogs.Common { @@ -19,8 +22,9 @@ public class AzureBlobLeaseManager public const int MaxRenewPeriodInSeconds = 60; // The lease will be renewed with a short interval before the the lease expires public const int OverlapRenewPeriodInSeconds = 20; - private BlobRequestOptions _blobRequestOptions; + //private BlobRequestOptions _blobRequestOptions; private readonly ILogger _logger; + private readonly BlobLeaseService _blobLeaseService; public AzureBlobLeaseManager(ILogger logger, BlobRequestOptions blobRequestOptions = null) { @@ -39,6 +43,8 @@ public AzureBlobLeaseManager(ILogger logger, BlobRequestO /// True if the lease was acquired. public AzureBlobLockResult AcquireLease(CloudBlob blob, CancellationToken token) { + //this stuff just checks if the blob is already locked or the token is cancelled + //tryacquareasync already does that blob.FetchAttributes(); if (token.IsCancellationRequested || blob.Properties.LeaseStatus == LeaseStatus.Locked) { @@ -48,10 +54,26 @@ public AzureBlobLockResult AcquireLease(CloudBlob blob, CancellationToken token) blob.Properties.LeaseStatus); return AzureBlobLockResult.FailedLockResult(blob); } + + //this part does the lease var proposedLeaseId = Guid.NewGuid().ToString(); var leaseId = blob.AcquireLease(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), proposedLeaseId); var lockResult = new AzureBlobLockResult(blob: blob, lockIsTaken: true, leaseId: leaseId, linkToken: token); + + + + + + + + + + + + + + //start a task that will renew the lease until the token is cancelled or the Release methods was invoked var renewStatusTask = new Task( (lockresult) => { diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs index e44e0026ec..8ae29dd2d5 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs @@ -1,9 +1,9 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Threading; -using Microsoft.WindowsAzure.Storage.Blob; +//using Microsoft.WindowsAzure.Storage.Blob; namespace Stats.AzureCdnLogs.Common { diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index 2ee3163ba9..eaa0909376 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -8,8 +8,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +//using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; using ICSharpCode.SharpZipLib.GZip; namespace Stats.AzureCdnLogs.Common.Collect @@ -28,11 +29,11 @@ public class AzureStatsLogSource : ILogSource private string _deadletterContainerName = "-deadletter"; private string _archiveContainerName = "-archive"; - private CloudStorageAccount _azureAccount; - private AzureBlobLeaseManager _blobLeaseManager; - private CloudBlobContainer _container; - private CloudBlobClient _blobClient; - private BlobRequestOptions _blobRequestOptions; + private BlobServiceClient _blobServiceClient; + //private AzureBlobLeaseManager _blobLeaseManager; + private BlobContainerClient _container; + //private CloudBlobClient _blobClient; + //private BlobRequestOptions _blobRequestOptions; private readonly ILogger _logger; /// @@ -40,17 +41,18 @@ public class AzureStatsLogSource : ILogSource /// /// The connection string for the Azure account. /// The container name. - public AzureStatsLogSource(CloudStorageAccount storageAccount, + public AzureStatsLogSource(BlobServiceClient blobServiceClient, string containerName, int azureServerTimeoutInSeconds, AzureBlobLeaseManager blobLeaseManager, ILogger logger) { - _azureAccount = storageAccount; - _blobClient = _azureAccount.CreateCloudBlobClient(); - _container = _blobClient.GetContainerReference(containerName); + _blobServiceClient = blobServiceClient; + //_blobClient = _azureBlobServiceClient.CreateCloudBlobClient(); + _container = _blobServiceClient.GetBlobContainerClient(containerName); + _blobRequestOptions = new BlobRequestOptions(); - _blobRequestOptions.ServerTimeout = TimeSpan.FromSeconds(azureServerTimeoutInSeconds); + _blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager)); _deadletterContainerName = $"{containerName}-deadletter"; _archiveContainerName = $"{containerName}-archive"; @@ -269,9 +271,9 @@ private async Task GetBlobAsync(Uri blobUri) } /// - /// Copy the blob from souurce to the destination + /// Copy the blob from source to the destination /// - /// The source blobLock as was taken at the begining of the operation. + /// The source blobLock as was taken at the beginning of the operation. /// The destination Container. /// private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlobInformation, CloudBlobContainer destinationContainer, CancellationToken token) diff --git a/src/Stats.AzureCdnLogs.Common/Stats.AzureCdnLogs.Common.csproj b/src/Stats.AzureCdnLogs.Common/Stats.AzureCdnLogs.Common.csproj index 59e42f7167..4a42627cc2 100644 --- a/src/Stats.AzureCdnLogs.Common/Stats.AzureCdnLogs.Common.csproj +++ b/src/Stats.AzureCdnLogs.Common/Stats.AzureCdnLogs.Common.csproj @@ -22,8 +22,9 @@ + - + \ No newline at end of file diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index 78c4c3fef8..7f7bad9979 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -11,8 +11,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage; using NuGet.Jobs; +using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; using Stats.AzureCdnLogs.Common.Collect; @@ -42,14 +43,14 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService>()); var source = new AzureStatsLogSource( - ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource), + ValidateAzureBlobServiceClient(_configuration.AzureAccountConnectionStringSource), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess, blobLeaseManager, serviceProvider.GetRequiredService>()); var dest = new AzureStatsLogDestination( - ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination), + ValidateAzureBlobServiceClient(_configuration.AzureAccountConnectionStringDestination), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); @@ -111,19 +112,21 @@ public override async Task Run() } } - private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount) + private static BlobServiceClient ValidateAzureBlobServiceClient(string blobServiceClient) { - if (string.IsNullOrEmpty(cloudStorageAccount)) + if (string.IsNullOrEmpty(blobServiceClient)) { - throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is not defined."); + throw new ArgumentException("Job parameter for Azure CDN Blob Service Client is not defined."); } - CloudStorageAccount account; - if (CloudStorageAccount.TryParse(cloudStorageAccount, out account)) + try + { + return new BlobServiceClient(blobServiceClient); + } + catch (Exception ex) { - return account; + throw new ArgumentException("Job parameter for Azure CDN Blob Service Client is invalid.", ex); } - throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is invalid."); } protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot) From 572ca1d01d515466626108322bd1f67b393a6352 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 16 Jan 2025 09:25:47 -0800 Subject: [PATCH 15/44] Changes --- .../AzureHelpers/AzureBlobLockResult.cs | 7 +-- .../Collect/AzureStatsLogSource.cs | 52 ++++++++++--------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs index 8ae29dd2d5..cc139a44f2 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs @@ -3,6 +3,7 @@ using System; using System.Threading; +using Azure.Storage.Blobs; //using Microsoft.WindowsAzure.Storage.Blob; namespace Stats.AzureCdnLogs.Common @@ -13,7 +14,7 @@ public class AzureBlobLockResult : IDisposable public string LeaseId { get; } - public CloudBlob Blob { get; } + public BlobClient Blob { get; } /// /// It will be cancelled when the renew task could not renew the lease. @@ -21,7 +22,7 @@ public class AzureBlobLockResult : IDisposable /// public CancellationTokenSource BlobOperationToken { get; } - public AzureBlobLockResult(CloudBlob blob, bool lockIsTaken, string leaseId, CancellationToken linkToken) + public AzureBlobLockResult(BlobClient blob, bool lockIsTaken, string leaseId, CancellationToken linkToken) { Blob = blob ?? throw new ArgumentNullException(nameof(blob)); LockIsTaken = lockIsTaken; @@ -30,7 +31,7 @@ public AzureBlobLockResult(CloudBlob blob, bool lockIsTaken, string leaseId, Can LeaseId = leaseId; } - public static AzureBlobLockResult FailedLockResult(CloudBlob blob) + public static AzureBlobLockResult FailedLockResult(BlobClient blob) { return new AzureBlobLockResult(blob: blob, lockIsTaken: false, leaseId: null, linkToken: CancellationToken.None); } diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index eaa0909376..ae5e4c8446 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -12,6 +12,7 @@ //using Microsoft.WindowsAzure.Storage.Blob; using Azure.Storage.Blobs; using ICSharpCode.SharpZipLib.GZip; +using Azure.Storage.Blobs.Models; namespace Stats.AzureCdnLogs.Common.Collect { @@ -30,7 +31,7 @@ public class AzureStatsLogSource : ILogSource private string _deadletterContainerName = "-deadletter"; private string _archiveContainerName = "-archive"; private BlobServiceClient _blobServiceClient; - //private AzureBlobLeaseManager _blobLeaseManager; + private AzureBlobLeaseManager _blobLeaseManager; private BlobContainerClient _container; //private CloudBlobClient _blobClient; //private BlobRequestOptions _blobRequestOptions; @@ -51,8 +52,6 @@ public AzureStatsLogSource(BlobServiceClient blobServiceClient, //_blobClient = _azureBlobServiceClient.CreateCloudBlobClient(); _container = _blobServiceClient.GetBlobContainerClient(containerName); - _blobRequestOptions = new BlobRequestOptions(); - _blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager)); _deadletterContainerName = $"{containerName}-deadletter"; _archiveContainerName = $"{containerName}-archive"; @@ -89,7 +88,7 @@ public async Task> GetFilesAsync(int maxResults, CancellationTo blobListingDetails: BlobListingDetails.Metadata, maxResults: null, currentToken: continuationToken, - options: _blobRequestOptions, + options: null, operationContext: null, cancellationToken: token); @@ -108,13 +107,14 @@ public async Task> GetFilesAsync(int maxResults, CancellationTo "Found blob {BlobUrl}, determining lease status...", blobItem.Uri); - var blob = (CloudBlob)blobItem; - if (blob.Properties.LeaseStatus != LeaseStatus.Unlocked) + var blob = (BlobClient)blobItem; + BlobProperties properties = await blob.GetPropertiesAsync(); + if (properties.LeaseStatus != LeaseStatus.Unlocked) { _logger.LogInformation( "Skipping blob {BlobUrl} as its lease status is not unlocked: {LeaseStatus}", blob.Uri, - blob.Properties.LeaseStatus); + properties.LeaseStatus); continue; } @@ -151,7 +151,9 @@ public async Task OpenReadAsync(Uri blobUri, ContentType contentType, Ca _logger.LogInformation("OpenReadAsync: The blob was not found. Blob {BlobUri}", blobUri.AbsoluteUri); return null; } - _logger.LogInformation("Opening blob {Filename}, {BlobSize} bytes", blob.Name, blob.Properties.Length); + + BlobProperties properties = await blob.GetPropertiesAsync(); + _logger.LogInformation("Opening blob {Filename}, {BlobSize} bytes", blob.Name, properties.ContentLength); var inputRawStream = await blob.OpenReadAsync(); switch (contentType) { @@ -241,7 +243,7 @@ public async Task TryCleanAsync(AzureBlobLockResult blobLo // The operation will throw if the lease does not match bool deleteResult = await sourceBlob.DeleteIfExistsAsync(deleteSnapshotsOption: DeleteSnapshotsOption.IncludeSnapshots, accessCondition: accessCondition, - options: _blobRequestOptions, + options: null, operationContext: null, cancellationToken: token); _logger.LogInformation("CleanAsync: Blob {Blob} was deleted {DeletedResult}. The leaseId: {LeaseId}", blobLock.Blob.Uri, deleteResult, blobLock.LeaseId); @@ -257,12 +259,12 @@ public async Task TryCleanAsync(AzureBlobLockResult blobLo } } - private async Task GetBlobAsync(Uri blobUri) + private async Task GetBlobAsync(Uri blobUri) { try { - var blob = await _blobClient.GetBlobReferenceFromServerAsync(blobUri, accessCondition: null, options: _blobRequestOptions, operationContext: null); - return blob as CloudBlob; + var blob = await _blobClient.GetBlobReferenceFromServerAsync(blobUri, accessCondition: null, options: null, operationContext: null); + return blob as BlobClient; } catch (Exception) { @@ -276,7 +278,7 @@ private async Task GetBlobAsync(Uri blobUri) /// The source blobLock as was taken at the beginning of the operation. /// The destination Container. /// - private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlobInformation, CloudBlobContainer destinationContainer, CancellationToken token) + private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlobInformation, BlobContainerClient destinationContainer, CancellationToken token) { if (token.IsCancellationRequested) { @@ -285,7 +287,7 @@ private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlob } //just get a reference to the future blob - var destinationBlob = destinationContainer.GetBlobReference(GetBlobNameFromUri(sourceBlobInformation.Blob.Uri)); + var destinationBlob = destinationContainer.GetBlobClient(GetBlobNameFromUri(sourceBlobInformation.Blob.Uri)); try { if (!await destinationBlob.ExistsAsync(token)) @@ -305,23 +307,23 @@ private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlob { destinationBlob.ReleaseLease(destinationAccessCondition); } - catch (StorageException) + catch (Azure.RequestFailedException) { // do not do anything the lease will be released anyway } return result; } } - catch (StorageException exception) + catch (Azure.RequestFailedException exception) { _logger.LogCritical(LogEvents.FailedBlobCopy, exception, "CopyBlobToContainerAsync: Blob Copy Failed. SourceUri: {SourceUri}. DestinationUri {DestinationUri}", sourceBlobInformation.Blob.Uri, destinationBlob.Uri); return false; } } - private async Task TryCopyInternalAsync(CloudBlob sourceBlob, - CloudBlob destinationBlob, - CloudBlobContainer destinationContainer, + private async Task TryCopyInternalAsync(BlobClient sourceBlob, + BlobClient destinationBlob, + BlobContainerClient destinationContainer, AccessCondition destinationAccessCondition = null) { var copySourceblobUri = sourceBlob.Uri; @@ -330,19 +332,21 @@ private async Task TryCopyInternalAsync(CloudBlob sourceBlob, copySourceblobUri = new Uri(sourceBlob.Uri.AbsoluteUri + sourceBlob.ServiceClient.Credentials.SASToken); } - await destinationBlob.StartCopyAsync(copySourceblobUri, + await destinationBlob.StartCopyFromUriAsync(copySourceblobUri, sourceAccessCondition: null, destAccessCondition: destinationAccessCondition, options: null, operationContext: null); //round-trip to the server and get the information - destinationBlob = (CloudBlob)destinationContainer.GetBlobReferenceFromServer(GetBlobNameFromUri(sourceBlob.Uri)); + destinationBlob = destinationContainer.GetBlobClient(GetBlobNameFromUri(sourceBlob.Uri)); + + BlobProperties properties = await destinationBlob.GetPropertiesAsync(); - while (destinationBlob.CopyState.Status == CopyStatus.Pending) + while (properties.CopyStatus == CopyStatus.Pending) { Task.Delay(TimeSpan.FromSeconds(1)).Wait(); - destinationBlob = (CloudBlob)destinationContainer.GetBlobReference(GetBlobNameFromUri(sourceBlob.Uri)); + destinationBlob = destinationContainer.GetBlobClient(GetBlobNameFromUri(sourceBlob.Uri)); } return true; } @@ -352,7 +356,7 @@ private string GetBlobNameFromUri(Uri blobUri) return blobUri.Segments.LastOrDefault(); } - private async Task CreateContainerAsync(string containerName) + private async Task CreateContainerAsync(string containerName) { var container = _blobClient.GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); From 710e1787e42180b58646443859d052eeb78da375 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 16 Jan 2025 15:30:35 -0800 Subject: [PATCH 16/44] AzureStatsLogSource.cs has been updated --- .../Collect/AzureStatsLogSource.cs | 94 +++++++------------ 1 file changed, 33 insertions(+), 61 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index ae5e4c8446..4529c3e08b 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -13,6 +13,7 @@ using Azure.Storage.Blobs; using ICSharpCode.SharpZipLib.GZip; using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; namespace Stats.AzureCdnLogs.Common.Collect { @@ -76,39 +77,18 @@ public async Task> GetFilesAsync(int maxResults, CancellationTo maxResults, prefix); - BlobContinuationToken continuationToken = null; var result = new List(); - do - { - _logger.LogInformation("Finding next blobs segment..."); - - var segment = await _container.ListBlobsSegmentedAsync( - prefix: prefix, - useFlatBlobListing: true, - blobListingDetails: BlobListingDetails.Metadata, - maxResults: null, - currentToken: continuationToken, - options: null, - operationContext: null, - cancellationToken: token); - - continuationToken = segment.ContinuationToken; - - _logger.LogInformation("Found next blobs segment, finding blobs with unlocked lease status..."); - - foreach (var blobItem in segment.Results) + await foreach (var blobItem in _container.GetBlobsAsync(prefix:prefix, cancellationToken:token)) { + + if (result.Count >= maxResults) { - if (result.Count >= maxResults) - { - break; - } + break; + } - _logger.LogInformation( - "Found blob {BlobUrl}, determining lease status...", - blobItem.Uri); + _logger.LogInformation("Found blob {BlobName}, determining lease status...", blobItem.Name); - var blob = (BlobClient)blobItem; - BlobProperties properties = await blob.GetPropertiesAsync(); + var blob = _container.GetBlobClient(blobItem.Name); + BlobProperties properties = await blob.GetPropertiesAsync(cancellationToken: token); if (properties.LeaseStatus != LeaseStatus.Unlocked) { _logger.LogInformation( @@ -119,10 +99,8 @@ public async Task> GetFilesAsync(int maxResults, CancellationTo } result.Add(blob.Uri); - } } - while (continuationToken != null); - + _logger.LogInformation( "Found {Results} unlocked blobs with prefix {Prefix}", result.Count, @@ -190,8 +168,8 @@ public async Task TakeLockAsync(Uri blobUri, CancellationTo /// /// Release the lock on the blob. /// - /// The blobLock as was taken at the begining of the operation. - /// A token to be used for cancellation. For this implemention the token is ignored. + /// The blobLock as was taken at the beginning of the operation. + /// A token to be used for cancellation. For this implementation the token is ignored. /// True if the lease was released or the blob does not exist. public async Task TryReleaseLockAsync(AzureBlobLockResult blobLock, CancellationToken token) { @@ -239,12 +217,10 @@ public async Task TryCleanAsync(AzureBlobLockResult blobLo if (await CopyBlobToContainerAsync(blobLock, archiveTargetContainer, token)) { _logger.LogInformation("CleanAsync: Blob {Blob} was copied to container {Container}", blobLock.Blob.Uri, archiveTargetContainerName); - var accessCondition = new AccessCondition { LeaseId = blobLock.LeaseId }; + var blobRequestConditions = new BlobRequestConditions { LeaseId = blobLock.LeaseId }; // The operation will throw if the lease does not match - bool deleteResult = await sourceBlob.DeleteIfExistsAsync(deleteSnapshotsOption: DeleteSnapshotsOption.IncludeSnapshots, - accessCondition: accessCondition, - options: null, - operationContext: null, + bool deleteResult = await sourceBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, + conditions: blobRequestConditions, cancellationToken: token); _logger.LogInformation("CleanAsync: Blob {Blob} was deleted {DeletedResult}. The leaseId: {LeaseId}", blobLock.Blob.Uri, deleteResult, blobLock.LeaseId); return new AsyncOperationResult(deleteResult, null); @@ -263,8 +239,9 @@ private async Task GetBlobAsync(Uri blobUri) { try { - var blob = await _blobClient.GetBlobReferenceFromServerAsync(blobUri, accessCondition: null, options: null, operationContext: null); - return blob as BlobClient; + var blobClient = new BlobClient(blobUri); + var properties = await blobClient.GetPropertiesAsync(); + return blobClient; } catch (Exception) { @@ -299,13 +276,14 @@ private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlob _logger.LogInformation("CopyBlobToContainerAsync: Blob already exists DestinationUri {DestinationUri}.", destinationBlob.Uri); // Overwrite - var lease = destinationBlob.AcquireLease(TimeSpan.FromSeconds(CopyBlobLeaseTimeInSeconds), sourceBlobInformation.LeaseId); - var destinationAccessCondition = new AccessCondition { LeaseId = lease }; - await destinationBlob.DeleteAsync(deleteSnapshotsOption: DeleteSnapshotsOption.IncludeSnapshots, accessCondition: destinationAccessCondition, options: null, operationContext: null); - var result = await TryCopyInternalAsync(sourceBlobInformation.Blob, destinationBlob, destinationContainer, destinationAccessCondition: destinationAccessCondition); + var leaseClient = destinationBlob.GetBlobLeaseClient(); + var leaseResponse = await leaseClient.AcquireAsync(TimeSpan.FromSeconds(CopyBlobLeaseTimeInSeconds), cancellationToken: token); + var destinationAccessCondition = new BlobRequestConditions { LeaseId = leaseResponse.Value.LeaseId }; + await destinationBlob.DeleteAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: destinationAccessCondition, cancellationToken: token); + var result = await TryCopyInternalAsync(sourceBlobInformation.Blob, destinationBlob, destinationContainer); try { - destinationBlob.ReleaseLease(destinationAccessCondition); + await leaseClient.ReleaseAsync(destinationAccessCondition); } catch (Azure.RequestFailedException) { @@ -323,20 +301,11 @@ private async Task CopyBlobToContainerAsync(AzureBlobLockResult sourceBlob private async Task TryCopyInternalAsync(BlobClient sourceBlob, BlobClient destinationBlob, - BlobContainerClient destinationContainer, - AccessCondition destinationAccessCondition = null) + BlobContainerClient destinationContainer) { - var copySourceblobUri = sourceBlob.Uri; - if (sourceBlob.ServiceClient.Credentials.IsSAS) - { - copySourceblobUri = new Uri(sourceBlob.Uri.AbsoluteUri + sourceBlob.ServiceClient.Credentials.SASToken); - } + var copySourceBlobUri = sourceBlob.Uri; - await destinationBlob.StartCopyFromUriAsync(copySourceblobUri, - sourceAccessCondition: null, - destAccessCondition: destinationAccessCondition, - options: null, - operationContext: null); + await destinationBlob.StartCopyFromUriAsync(copySourceBlobUri); //round-trip to the server and get the information destinationBlob = destinationContainer.GetBlobClient(GetBlobNameFromUri(sourceBlob.Uri)); @@ -345,12 +314,15 @@ await destinationBlob.StartCopyFromUriAsync(copySourceblobUri, while (properties.CopyStatus == CopyStatus.Pending) { + properties = await destinationBlob.GetPropertiesAsync(); Task.Delay(TimeSpan.FromSeconds(1)).Wait(); - destinationBlob = destinationContainer.GetBlobClient(GetBlobNameFromUri(sourceBlob.Uri)); } - return true; + + BlobProperties resultProperties = await destinationBlob.GetPropertiesAsync(); + return resultProperties.CopyStatus == CopyStatus.Success; ; } + private string GetBlobNameFromUri(Uri blobUri) { return blobUri.Segments.LastOrDefault(); @@ -358,7 +330,7 @@ private string GetBlobNameFromUri(Uri blobUri) private async Task CreateContainerAsync(string containerName) { - var container = _blobClient.GetContainerReference(containerName); + var container = _blobServiceClient.GetBlobContainerClient(containerName); await container.CreateIfNotExistsAsync(); return container; } From 626d2e6fa6b732be985f7938eab35776d7924dfe Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 17 Jan 2025 13:56:39 -0800 Subject: [PATCH 17/44] Migrated more files and tests --- .../AzureHelpers/AzureBlobLeaseManager.cs | 126 +++++++----------- .../Collect/AzureStatsLogSource.cs | 7 +- src/Stats.CDNLogsSanitizer/Job.cs | 12 +- src/Stats.CDNLogsSanitizer/Processor.cs | 8 +- .../AzureBlobLeaseManagerTests.cs | 4 +- .../AzureBlobLockResultTests.cs | 9 +- .../ChinaCollectorTests.cs | 3 +- .../JobTests.cs | 12 +- 8 files changed, 78 insertions(+), 103 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index bc66d666d2..2990c5bc49 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -8,6 +8,7 @@ using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common.Collect; using NuGet.Services.Storage; +using Azure.Storage.Blobs.Specialized; //using Microsoft.WindowsAzure.Storage; //using Microsoft.WindowsAzure.Storage.Blob; @@ -26,9 +27,8 @@ public class AzureBlobLeaseManager private readonly ILogger _logger; private readonly BlobLeaseService _blobLeaseService; - public AzureBlobLeaseManager(ILogger logger, BlobRequestOptions blobRequestOptions = null) + public AzureBlobLeaseManager(ILogger logger) { - _blobRequestOptions = blobRequestOptions; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -41,98 +41,70 @@ public AzureBlobLeaseManager(ILogger logger, BlobRequestO /// A token to cancel the operation. /// The renew task. /// True if the lease was acquired. - public AzureBlobLockResult AcquireLease(CloudBlob blob, CancellationToken token) + public async Task AcquireLease(BlobClient blob, CancellationToken token) { - //this stuff just checks if the blob is already locked or the token is cancelled - //tryacquareasync already does that - blob.FetchAttributes(); - if (token.IsCancellationRequested || blob.Properties.LeaseStatus == LeaseStatus.Locked) + try { - _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}, Cancellation status {IsCancellationRequested}, BlobLeaseStatus {BlobLeaseStatus}.", - blob.Uri.AbsoluteUri, - token.IsCancellationRequested, - blob.Properties.LeaseStatus); - return AzureBlobLockResult.FailedLockResult(blob); - } - - //this part does the lease - var proposedLeaseId = Guid.NewGuid().ToString(); - var leaseId = blob.AcquireLease(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), proposedLeaseId); - var lockResult = new AzureBlobLockResult(blob: blob, lockIsTaken: true, leaseId: leaseId, linkToken: token); - - - - - - - - - - - - - + var leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), token); + if (!leaseResult.IsSuccess) + { + _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}.", blob.Uri.AbsoluteUri); + return AzureBlobLockResult.FailedLockResult(blob); + } + var leaseId = leaseResult.LeaseId; + var lockResult = new AzureBlobLockResult(blob, true, leaseId, token); - //start a task that will renew the lease until the token is cancelled or the Release methods was invoked - var renewStatusTask = new Task( (lockresult) => + // Start a task that will renew the lease until the token is cancelled or the Release method is invoked + _ = Task.Run(async () => { - var blobLockResult = (AzureBlobLockResult)lockresult; - _logger.LogInformation("RenewLeaseTask: Started for BlobUri {BlobUri}. ThreadId {ThreadId}. IsCancellationRequested {IsCancellationRequested}. LeaseId {LeaseId}", - blob.Uri.AbsoluteUri, - Thread.CurrentThread.ManagedThreadId, - blobLockResult.BlobOperationToken.IsCancellationRequested, - blobLockResult.LeaseId); - - int sleepBeforeRenewInSeconds = MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds < 0 ? MaxRenewPeriodInSeconds : MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds; - if (!blobLockResult.BlobOperationToken.IsCancellationRequested) + while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) { - while (!blobLockResult.BlobOperationToken.Token.IsCancellationRequested) + try { - Thread.Sleep(sleepBeforeRenewInSeconds * 1000); - - //it will renew the lease only if the lease was not explicitly released - try - { - if (!blobLockResult.Blob.Exists()) - { - blobLockResult.BlobOperationToken.Cancel(); - break; - } - AccessCondition acc = new AccessCondition { LeaseId = blobLockResult.LeaseId }; - blob.RenewLease(accessCondition: acc, options: _blobRequestOptions, operationContext: null); - _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", - blob.Uri.AbsoluteUri, - blobLockResult.LeaseId); - } - catch (StorageException exception) - { - _logger.LogWarning(LogEvents.FailedBlobLease, exception, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. ExpectedLeaseId {LeaseId}. CurrentLeaseId {CurrentLeaseId}.", - blob.Uri.AbsoluteUri, - leaseId, - blobLockResult.LeaseId); - blobLockResult.BlobOperationToken.Cancel(); - break; - } + await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); + await _blobLeaseService.RenewAsync(blob.Name, leaseId, lockResult.BlobOperationToken.Token); + _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); + lockResult.BlobOperationToken.Cancel(); + break; } } - }, lockResult, TaskCreationOptions.LongRunning); - renewStatusTask.Start(); - return lockResult; + }, lockResult.BlobOperationToken.Token); + + return lockResult; + } + catch (Exception ex) + { + _logger.LogError(ex, "AcquireLeaseAsync: Failed to acquire lease for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); + return AzureBlobLockResult.FailedLockResult(blob); + } + } public async Task TryReleaseLockAsync(AzureBlobLockResult releaseLock) { try { - AccessCondition acc = new AccessCondition(); - acc.LeaseId = releaseLock.LeaseId; if(await releaseLock.Blob.ExistsAsync()) { - await releaseLock.Blob.ReleaseLeaseAsync(acc, options: _blobRequestOptions, operationContext: null); - releaseLock.BlobOperationToken.Cancel(); - _logger.LogInformation("ReleaseLockAsync: ReleaseLeaseStatus: {LeaseReleased} on the {BlobUri}.", true, releaseLock.Blob.Uri); - return new AsyncOperationResult(true, null); + bool releaseResult = await _blobLeaseService.ReleaseAsync(releaseLock.Blob.Name, releaseLock.LeaseId, releaseLock.BlobOperationToken.Token); + if (releaseResult) + { + _logger.LogInformation("ReleaseLockAsync: ReleaseLeaseStatus: {LeaseReleased} on the {BlobUri}.", true, releaseLock.Blob.Uri); + return new AsyncOperationResult(true, null); + } + else + { + return new AsyncOperationResult(false, null); + } } else { diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index 4529c3e08b..a0af7f2da4 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -8,8 +8,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -//using Microsoft.WindowsAzure.Storage; -//using Microsoft.WindowsAzure.Storage.Blob; using Azure.Storage.Blobs; using ICSharpCode.SharpZipLib.GZip; using Azure.Storage.Blobs.Models; @@ -34,8 +32,6 @@ public class AzureStatsLogSource : ILogSource private BlobServiceClient _blobServiceClient; private AzureBlobLeaseManager _blobLeaseManager; private BlobContainerClient _container; - //private CloudBlobClient _blobClient; - //private BlobRequestOptions _blobRequestOptions; private readonly ILogger _logger; /// @@ -50,7 +46,6 @@ public AzureStatsLogSource(BlobServiceClient blobServiceClient, ILogger logger) { _blobServiceClient = blobServiceClient; - //_blobClient = _azureBlobServiceClient.CreateCloudBlobClient(); _container = _blobServiceClient.GetBlobContainerClient(containerName); _blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager)); @@ -162,7 +157,7 @@ public async Task TakeLockAsync(Uri blobUri, CancellationTo { return AzureBlobLockResult.FailedLockResult(blob); } - return _blobLeaseManager.AcquireLease(blob, token); + return await _blobLeaseManager.AcquireLease(blob, token); } /// diff --git a/src/Stats.CDNLogsSanitizer/Job.cs b/src/Stats.CDNLogsSanitizer/Job.cs index cf50f81eaa..e73a397991 100644 --- a/src/Stats.CDNLogsSanitizer/Job.cs +++ b/src/Stats.CDNLogsSanitizer/Job.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -11,7 +11,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage; +using Azure; +using Azure.Storage.Blobs; using NuGet.Jobs; using Stats.AzureCdnLogs.Common; using Stats.AzureCdnLogs.Common.Collect; @@ -79,15 +81,15 @@ public override async Task Run() } } - private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount) + private static BlobServiceClient ValidateAzureCloudStorageAccount(string cloudStorageAccount) { if (string.IsNullOrEmpty(cloudStorageAccount)) { throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is not defined."); } - CloudStorageAccount account; - if (CloudStorageAccount.TryParse(cloudStorageAccount, out account)) + BlobServiceClient account; + if (BlobServiceClient.TryParse(cloudStorageAccount, out account)) { return account; } diff --git a/src/Stats.CDNLogsSanitizer/Processor.cs b/src/Stats.CDNLogsSanitizer/Processor.cs index 8cb3855d9d..6b15716a00 100644 --- a/src/Stats.CDNLogsSanitizer/Processor.cs +++ b/src/Stats.CDNLogsSanitizer/Processor.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -9,7 +9,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage; +using Azure; +using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; using Stats.AzureCdnLogs.Common.Collect; @@ -132,7 +134,7 @@ public void ProcessStream(Stream sourceStream, Stream targetStream) _logger.LogInformation("ProcessStream: Finished writting to the destination stream."); } } - catch (StorageException exception) + catch (RequestFailedException exception) { _logger.LogCritical(LogEvents.FailedToProcessStream, exception, "ProcessStream: An exception while processing the stream."); throw; diff --git a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs index 0a74086049..7ddeb09349 100644 --- a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs +++ b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -12,7 +12,7 @@ public class AzureBlobLeaseManagerTests [Fact] public void ConstructorNullArgumentTest() { - Assert.Throws(() => new AzureBlobLeaseManager(null, null)); + Assert.Throws(() => new AzureBlobLeaseManager(null)); } [Fact] diff --git a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs index fe303f331c..c618ac31e9 100644 --- a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs +++ b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs @@ -1,9 +1,10 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Threading; -using Microsoft.WindowsAzure.Storage.Blob; +//using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; using Xunit; @@ -28,7 +29,7 @@ public void VerifyThatTheTokenRespectsExternalCancellation() // Arrange var cts = new CancellationTokenSource(); var testAzureBlobLockResult = new AzureBlobLockResult( - blob: new CloudBlob(new Uri("https://test/container")), + blob: new BlobClient(new Uri("https://test/container")), lockIsTaken: false, leaseId: string.Empty, linkToken: cts.Token); @@ -47,7 +48,7 @@ public void VerifyThatTheTokenCancellationDoesNotAffectExternalLinkedToken() var cts = new CancellationTokenSource(); var externalToken = cts.Token; var testAzureBlobLockResult = new AzureBlobLockResult( - blob: new CloudBlob(new Uri("https://test/container")), + blob: new BlobClient(new Uri("https://test/container")), lockIsTaken: false, leaseId: string.Empty, linkToken: externalToken); diff --git a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs index afe299e6e3..8ea6242ccd 100644 --- a/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs +++ b/tests/Tests.Stats.CollectAzureChinaCDNLogs/ChinaCollectorTests.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Azure.Storage.Blobs; using Moq; using Stats.AzureCdnLogs.Common; using Stats.AzureCdnLogs.Common.Collect; @@ -256,7 +257,7 @@ private static Mock SetupSource(string content) sourceMock .Setup(s => s.TakeLockAsync(sourceUri, It.IsAny())) - .ReturnsAsync(new AzureBlobLockResult(new Microsoft.WindowsAzure.Storage.Blob.CloudBlob(sourceUri), true, "foo", CancellationToken.None)); + .ReturnsAsync(new AzureBlobLockResult(new BlobClient(sourceUri), true, "foo", CancellationToken.None)); sourceMock .Setup(s => s.OpenReadAsync(sourceUri, It.IsAny(), It.IsAny())) diff --git a/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs b/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs index 968201ec1a..77491cf5bb 100644 --- a/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs +++ b/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -6,7 +6,9 @@ using System.ComponentModel.Design; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Microsoft.WindowsAzure.Storage; +//using Microsoft.WindowsAzure.Storage; +using Azure.Storage.Blobs; +using Azure; using Moq; using Stats.AzureCdnLogs.Common.Collect; using Stats.AzureCdnLogs.Common; @@ -34,7 +36,7 @@ public void InitFailsWithInvalidAccount() var job = new Job(); var configuration = GetDefaultConfiguration(); - Assert.ThrowsAny(() => job.InitializeJobConfiguration(GetMockServiceProvider(configuration))); + Assert.ThrowsAny(() => job.InitializeJobConfiguration(GetMockServiceProvider(configuration))); } [Theory] @@ -43,13 +45,13 @@ public void InitFailsWithInvalidAccount() [InlineData("AzureAccountConnectionStringDestination", null, typeof(ArgumentException))] [InlineData("AzureContainerNameDestination", null, typeof(ArgumentNullException))] [InlineData("AzureContainerNameSource", null, typeof(ArgumentNullException))] - [InlineData("DestinationFilePrefix", null, typeof(StorageException))] + [InlineData("DestinationFilePrefix", null, typeof(RequestFailedException))] // empty values [InlineData("AzureAccountConnectionStringSource", "", typeof(ArgumentException))] [InlineData("AzureAccountConnectionStringDestination", "", typeof(ArgumentException))] [InlineData("AzureContainerNameDestination", "", typeof(ArgumentException))] [InlineData("AzureContainerNameSource", "", typeof(ArgumentException))] - [InlineData("DestinationFilePrefix", "", typeof(StorageException))] + [InlineData("DestinationFilePrefix", "", typeof(RequestFailedException))] public void InitMissingArgArguments(string property, object value, Type exceptionType) { var job = new Job(); From bb56055f26bb23922f8015d6d0c1734440134e0c Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 17 Jan 2025 16:18:28 -0800 Subject: [PATCH 18/44] Final fixes? --- src/Stats.CDNLogsSanitizer/Job.cs | 9 ++++++--- .../Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs | 14 ++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Stats.CDNLogsSanitizer/Job.cs b/src/Stats.CDNLogsSanitizer/Job.cs index e73a397991..3a8ffa60e5 100644 --- a/src/Stats.CDNLogsSanitizer/Job.cs +++ b/src/Stats.CDNLogsSanitizer/Job.cs @@ -88,12 +88,15 @@ private static BlobServiceClient ValidateAzureCloudStorageAccount(string cloudSt throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is not defined."); } - BlobServiceClient account; - if (BlobServiceClient.TryParse(cloudStorageAccount, out account)) + try { + var account = new BlobServiceClient(cloudStorageAccount); return account; } - throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is invalid."); + catch (Exception ex) + { + throw new ArgumentException("Job parameter for Azure CDN Cloud Storage Account is invalid.", ex); + } } protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot) diff --git a/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs b/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs index 4d87fbd648..e70073e3fa 100644 --- a/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs +++ b/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -7,7 +7,9 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage.Blob; +//using Microsoft.WindowsAzure.Storage.Blob; +using Azure.Storage.Blobs; +using Azure; using Microsoft.Extensions.Logging; using Stats.AzureCdnLogs.Common; using Stats.AzureCdnLogs.Common.Collect; @@ -67,7 +69,7 @@ public async Task TheProcessOfABlobIsStoppedOnCancellation() // Arrange // Set a source that returns 1 element to process var _logSource = new Mock(); - var dummyLockResult = new AzureBlobLockResult(new CloudBlob(new Uri("https://dummy/foo.gz")), true, "leaseid", CancellationToken.None); + var dummyLockResult = new AzureBlobLockResult(new BlobClient(new Uri("https://dummy/foo.gz")), true, "leaseid", CancellationToken.None); dummyLockResult.BlobOperationToken.Cancel(); _logSource.Setup(lS => lS.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new Uri[] { new Uri("https://dummy") }); @@ -99,7 +101,7 @@ public async Task IfTheSourceDoesNotHaveMoreElementsTheExecutionWillStop() // Arrange // Set a source that returns 1 element to process var uri = "https://dummy/foo.gz"; - var cb = new CloudBlob(new Uri(uri)); + var cb = new BlobClient(new Uri(uri)); var _logSource = new Mock(); var dummyLockResult = new AzureBlobLockResult(cb, false, "leaseid", CancellationToken.None); _logSource.Setup(lS => lS.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new Uri[] { new Uri(uri) }); @@ -132,7 +134,7 @@ public async Task VerifyCleanAsyncCallOnErrorIsInvokedWhenWriteAsyncThrows(bool // Arrange // Set a source that returns 1 element to process var _logSource = new Mock(); - var dummyLockResult = new AzureBlobLockResult(new CloudBlob(new Uri("https://dummy/foo.gz")), true, "leaseid", CancellationToken.None); + var dummyLockResult = new AzureBlobLockResult(new BlobClient(new Uri("https://dummy/foo.gz")), true, "leaseid", CancellationToken.None); _logSource.Setup(lS => lS.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new Uri[] { new Uri("https://dummy") }); _logSource.Setup(lS => lS.TakeLockAsync(It.IsAny(), It.IsAny())) @@ -169,7 +171,7 @@ public async Task WhenLockIsNotTakenTheExecutionDoesNotProceed() // Arrange int maxElementsToProcess = 1; var _logSource = new Mock(); - var dummyLockResult = new AzureBlobLockResult(new CloudBlob(new Uri("https://dummy/foo.gz")), false, "leaseid", CancellationToken.None); + var dummyLockResult = new AzureBlobLockResult(new BlobClient(new Uri("https://dummy/foo.gz")), false, "leaseid", CancellationToken.None); _logSource.Setup(lS => lS.GetFilesAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new Uri[] { new Uri("https://dummy/foo.gz"), new Uri("https://dummy/foo2.gz") }); _logSource.Setup(lS => lS.TakeLockAsync(It.IsAny(), It.IsAny())) From 28b8c530eb5a0c6a6ee3c1c42e280c3b82487d2a Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Tue, 21 Jan 2025 08:57:21 -0800 Subject: [PATCH 19/44] Fixed failing tests --- .../Collect/AzureStatsLogDestination.cs | 7 ++++++- .../Collect/AzureStatsLogSource.cs | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs index cbcd91ecd7..6cfb390ecc 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs @@ -27,7 +27,12 @@ public class AzureStatsLogDestination : ILogDestination public AzureStatsLogDestination(BlobServiceClient blobServiceClient, string containerName, ILogger logger) { - _blobServiceClient = blobServiceClient; + if (string.IsNullOrEmpty(containerName)) + { + if (containerName == null) throw new ArgumentNullException(nameof(containerName)); + else throw new ArgumentException(nameof(containerName)); + } + _blobServiceClient = blobServiceClient ?? throw new ArgumentNullException(nameof(logger)); _blobContainerClient = _blobServiceClient.GetBlobContainerClient(containerName); _blobContainerClient.CreateIfNotExists(); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index a0af7f2da4..c7c79509b3 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -45,7 +45,13 @@ public AzureStatsLogSource(BlobServiceClient blobServiceClient, AzureBlobLeaseManager blobLeaseManager, ILogger logger) { - _blobServiceClient = blobServiceClient; + _blobServiceClient = blobServiceClient ?? throw new ArgumentNullException(nameof(logger)); + + if (string.IsNullOrEmpty(containerName)) + { + if (containerName == null) throw new ArgumentNullException(nameof(containerName)); + else throw new ArgumentException(nameof(containerName)); + } _container = _blobServiceClient.GetBlobContainerClient(containerName); _blobLeaseManager = blobLeaseManager ?? throw new ArgumentNullException(nameof(blobLeaseManager)); From f6b50de53294eb99fee7a7f5e82c7b5d9ba91509 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 23 Jan 2025 10:44:39 -0800 Subject: [PATCH 20/44] Fixed blobleasemanager problem --- .../AzureHelpers/AzureBlobLeaseManager.cs | 5 +++-- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 2990c5bc49..62bb8ba7fa 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -25,11 +25,12 @@ public class AzureBlobLeaseManager public const int OverlapRenewPeriodInSeconds = 20; //private BlobRequestOptions _blobRequestOptions; private readonly ILogger _logger; - private readonly BlobLeaseService _blobLeaseService; + private BlobLeaseService _blobLeaseService; - public AzureBlobLeaseManager(ILogger logger) + public AzureBlobLeaseManager(ILogger logger, BlobServiceClient blobServiceClient, string containerName, string basePath) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath); } /// diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index 7f7bad9979..a70d8d26fd 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -40,17 +40,24 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) _configuration = serviceProvider.GetRequiredService>().Value; _executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds; - var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService>()); + var superstring = _configuration.AzureAccountConnectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + + var blobLeaseManager = new AzureBlobLeaseManager( + serviceProvider.GetRequiredService>(), + ValidateAzureBlobServiceClient(superstring), + _configuration.AzureContainerNameDestination, + superstring); + var source = new AzureStatsLogSource( - ValidateAzureBlobServiceClient(_configuration.AzureAccountConnectionStringSource), + ValidateAzureBlobServiceClient(superstring), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess, blobLeaseManager, serviceProvider.GetRequiredService>()); var dest = new AzureStatsLogDestination( - ValidateAzureBlobServiceClient(_configuration.AzureAccountConnectionStringDestination), + ValidateAzureBlobServiceClient(superstring), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); From e69fee8b39818320da40d22f37dfe069cdda1354 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 23 Jan 2025 11:32:27 -0800 Subject: [PATCH 21/44] Fixing --- src/Stats.CDNLogsSanitizer/Job.cs | 11 ++++++++--- .../AzureBlobLeaseManagerTests.cs | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Stats.CDNLogsSanitizer/Job.cs b/src/Stats.CDNLogsSanitizer/Job.cs index 3a8ffa60e5..0518683efe 100644 --- a/src/Stats.CDNLogsSanitizer/Job.cs +++ b/src/Stats.CDNLogsSanitizer/Job.cs @@ -48,17 +48,22 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) var logHeaderDelimiter = _configuration.LogHeaderDelimiter ?? throw new ArgumentNullException(nameof(_configuration.LogHeaderDelimiter)); _logHeaderMetadata = new LogHeaderMetadata(logHeader, logHeaderDelimiter); _blobPrefix = _configuration.BlobPrefix ; - var blobLeaseManager = new AzureBlobLeaseManager(serviceProvider.GetRequiredService>()); + var superstring = _configuration.AzureAccountConnectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + var blobLeaseManager = new AzureBlobLeaseManager( + serviceProvider.GetRequiredService>(), + ValidateAzureCloudStorageAccount(superstring), + _configuration.AzureContainerNameDestination, + superstring); var source = new AzureStatsLogSource( - ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringSource), + ValidateAzureCloudStorageAccount(superstring), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / _maxBlobsToProcess, blobLeaseManager, serviceProvider.GetRequiredService>()); var dest = new AzureStatsLogDestination( - ValidateAzureCloudStorageAccount(_configuration.AzureAccountConnectionStringDestination), + ValidateAzureCloudStorageAccount(superstring), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); diff --git a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs index 7ddeb09349..8b70c18219 100644 --- a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs +++ b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLeaseManagerTests.cs @@ -12,7 +12,7 @@ public class AzureBlobLeaseManagerTests [Fact] public void ConstructorNullArgumentTest() { - Assert.Throws(() => new AzureBlobLeaseManager(null)); + Assert.Throws(() => new AzureBlobLeaseManager(null, null, null, null)); } [Fact] From d613266f929dd0c04ab7548e055f78940d6f6604 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 23 Jan 2025 13:23:44 -0800 Subject: [PATCH 22/44] Test fixes --- .../AzureHelpers/AzureBlobLeaseManager.cs | 13 +++++++++++++ .../Collect/AzureStatsLogDestination.cs | 4 ++-- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 16 +++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 62bb8ba7fa..95d68a3ed1 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -30,6 +30,19 @@ public class AzureBlobLeaseManager public AzureBlobLeaseManager(ILogger logger, BlobServiceClient blobServiceClient, string containerName, string basePath) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + if (blobServiceClient == null) throw new ArgumentNullException(nameof(blobServiceClient)); + + if (string.IsNullOrEmpty(containerName)) + { + if (containerName == null) throw new ArgumentNullException(nameof(containerName)); + else throw new ArgumentException(nameof(containerName)); + } + + if (string.IsNullOrEmpty(basePath)) + { + if (containerName == null) throw new ArgumentNullException(nameof(basePath)); + else throw new ArgumentException(nameof(basePath)); + } _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath); } diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs index 6cfb390ecc..0da25ccfeb 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs @@ -32,10 +32,10 @@ public AzureStatsLogDestination(BlobServiceClient blobServiceClient, string cont if (containerName == null) throw new ArgumentNullException(nameof(containerName)); else throw new ArgumentException(nameof(containerName)); } - _blobServiceClient = blobServiceClient ?? throw new ArgumentNullException(nameof(logger)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _blobServiceClient = blobServiceClient ?? throw new ArgumentNullException(nameof(blobServiceClient)); _blobContainerClient = _blobServiceClient.GetBlobContainerClient(containerName); _blobContainerClient.CreateIfNotExists(); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index a70d8d26fd..82a7e96fe4 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -40,7 +40,21 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) _configuration = serviceProvider.GetRequiredService>().Value; _executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds; - var superstring = _configuration.AzureAccountConnectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + var superstring = _configuration.AzureAccountConnectionStringSource; + + + if (string.IsNullOrEmpty(_configuration.AzureAccountConnectionStringSource)) + { + throw new ArgumentException(nameof(superstring)); + } + + + if (string.IsNullOrEmpty(_configuration.AzureAccountConnectionStringDestination)) + { + throw new ArgumentException(nameof(superstring)); + } + + superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var blobLeaseManager = new AzureBlobLeaseManager( serviceProvider.GetRequiredService>(), From faca4bf2bbd1f9fab5d11573f043e488f2649665 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 23 Jan 2025 14:33:48 -0800 Subject: [PATCH 23/44] Buildfix --- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index 82a7e96fe4..7a6cd1f5fe 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -54,7 +54,7 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) throw new ArgumentException(nameof(superstring)); } - superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + superstring = superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var blobLeaseManager = new AzureBlobLeaseManager( serviceProvider.GetRequiredService>(), From c87896bb5e4b9c417331042d649d7c5474fe05b5 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 24 Jan 2025 09:36:55 -0800 Subject: [PATCH 24/44] Fixed cancellation problem --- .../AzureHelpers/AzureBlobLeaseManager.cs | 42 ++++++++++++------- .../AzureHelpers/AzureBlobLockResult.cs | 3 ++ 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 95d68a3ed1..5b0d49b5e3 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -68,31 +68,40 @@ public async Task AcquireLease(BlobClient blob, Cancellatio var leaseId = leaseResult.LeaseId; var lockResult = new AzureBlobLockResult(blob, true, leaseId, token); + var leaseRenewalCompletionSource = new TaskCompletionSource(); // Start a task that will renew the lease until the token is cancelled or the Release method is invoked _ = Task.Run(async () => { - while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) + try { - try + while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); - await _blobLeaseService.RenewAsync(blob.Name, leaseId, lockResult.BlobOperationToken.Token); - _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", - blob.Uri.AbsoluteUri, - leaseId); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. LeaseId {LeaseId}.", - blob.Uri.AbsoluteUri, - leaseId); - lockResult.BlobOperationToken.Cancel(); - break; + try + { + await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); + await _blobLeaseService.RenewAsync(blob.Name, leaseId, lockResult.BlobOperationToken.Token); + _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); + lockResult.BlobOperationToken.Cancel(); + break; + } } } + finally + { + leaseRenewalCompletionSource.SetResult(true); + } + }, lockResult.BlobOperationToken.Token); - + lockResult.LeaseRenewalCompletionSource = leaseRenewalCompletionSource.Task; return lockResult; } catch (Exception ex) @@ -113,6 +122,7 @@ public async Task TryReleaseLockAsync(AzureBlobLockResult if (releaseResult) { _logger.LogInformation("ReleaseLockAsync: ReleaseLeaseStatus: {LeaseReleased} on the {BlobUri}.", true, releaseLock.Blob.Uri); + await releaseLock.LeaseRenewalCompletionSource; // Added return new AsyncOperationResult(true, null); } else diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs index cc139a44f2..0bf6315396 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs @@ -3,6 +3,7 @@ using System; using System.Threading; +using System.Threading.Tasks; using Azure.Storage.Blobs; //using Microsoft.WindowsAzure.Storage.Blob; @@ -16,6 +17,8 @@ public class AzureBlobLockResult : IDisposable public BlobClient Blob { get; } + public Task LeaseRenewalCompletionSource { get; set; } + /// /// It will be cancelled when the renew task could not renew the lease. /// Operations can listen to this cancellation to stop execution once the lease could not be renewed. From 8e68ca6e3a1120061b4594835833c9167bbd92dc Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Tue, 28 Jan 2025 15:52:31 -0800 Subject: [PATCH 25/44] Latest changes --- .../BlobLeaseService.cs | 26 +++++++++--- .../AzureHelpers/AzureBlobLeaseManager.cs | 41 +++++++++---------- .../AzureHelpers/AzureBlobLockResult.cs | 3 -- .../Collect/AzureStatsLogSource.cs | 4 +- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 7 +++- 5 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/NuGet.Services.Storage/BlobLeaseService.cs b/src/NuGet.Services.Storage/BlobLeaseService.cs index e6c9c91f62..7da733ceb2 100644 --- a/src/NuGet.Services.Storage/BlobLeaseService.cs +++ b/src/NuGet.Services.Storage/BlobLeaseService.cs @@ -8,6 +8,7 @@ using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Specialized; +using Microsoft.Extensions.Logging; namespace NuGet.Services.Storage { @@ -20,11 +21,11 @@ public class BlobLeaseService : IBlobLeaseService { private static readonly TimeSpan MinLeaseTime = TimeSpan.FromSeconds(15); private static readonly TimeSpan MaxLeaseTime = TimeSpan.FromSeconds(60); - + private readonly ILogger _logger; private readonly BlobContainerClient _containerClient; private readonly string _basePath; - public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath) + public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath, ILogger logger) { if (blobServiceClient == null) { @@ -34,11 +35,11 @@ public BlobLeaseService(BlobServiceClient blobServiceClient, string containerNam { throw new ArgumentException("The container name must be provided.", nameof(containerName)); } - if (string.IsNullOrEmpty(basePath)) + if (basePath == null) { throw new ArgumentException("The base path must be provided.", nameof(basePath)); } - + _logger = logger; _containerClient = blobServiceClient.GetBlobContainerClient(containerName); _basePath = string.IsNullOrEmpty(basePath) ? string.Empty : basePath.TrimEnd('/') + '/'; } @@ -46,15 +47,22 @@ public BlobLeaseService(BlobServiceClient blobServiceClient, string containerNam public async Task TryAcquireAsync(string resourceName, TimeSpan leaseTime, CancellationToken cancellationToken) { var blob = GetBlob(resourceName); + try { - return await TryAcquireAsync(blob, leaseTime, cancellationToken); + var result = await TryAcquireAsync(blob, leaseTime, cancellationToken); + _logger.LogInformation("TryAcquireAsync: {resourceName} {leaseId} {leaseTime}", resourceName, result.LeaseId, leaseTime); + + return result; } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) { // The lease file does not exist. Try to create it and lease it. - return await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); + var result1 = await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); + _logger.LogInformation("TryAcquireAsync2: {resourceName} {leaseId}, {leaseTime}", resourceName, result1.LeaseId, leaseTime); + + return result1; } } @@ -64,6 +72,8 @@ public async Task ReleaseAsync(string resourceName, string leaseId, Cancel { var blob = GetBlob(resourceName); var leaseClient = blob.GetBlobLeaseClient(leaseId); + _logger.LogInformation("ReleaseAsync: {resourceName} {leaseId}", resourceName, leaseId); + await leaseClient.ReleaseAsync(conditions: null, cancellationToken: cancellationToken); return true; @@ -93,7 +103,10 @@ public async Task RenewAsync(string resourceName, string leaseI try { + _logger.LogInformation("RenewAsync: {resourceName} {leaseId}", resourceName, leaseId); + var lease = await leaseClient.RenewAsync(conditions: null, cancellationToken: cancellationToken); + return BlobLeaseResult.Success(lease); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict || ex.Status == (int)HttpStatusCode.PreconditionFailed) @@ -135,6 +148,7 @@ private async Task TryAcquireAsync(BlobClient blob, TimeSpan le { var leaseClient = blob.GetBlobLeaseClient(); var blobLease = await leaseClient.AcquireAsync(leaseTime, conditions: null, cancellationToken: cancellationToken); + return BlobLeaseResult.Success(blobLease); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 5b0d49b5e3..bf3f5d6574 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -38,12 +38,8 @@ public AzureBlobLeaseManager(ILogger logger, BlobServiceC else throw new ArgumentException(nameof(containerName)); } - if (string.IsNullOrEmpty(basePath)) - { - if (containerName == null) throw new ArgumentNullException(nameof(basePath)); - else throw new ArgumentException(nameof(basePath)); - } - _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath); + + _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath, logger); } /// @@ -55,32 +51,38 @@ public AzureBlobLeaseManager(ILogger logger, BlobServiceC /// A token to cancel the operation. /// The renew task. /// True if the lease was acquired. - public async Task AcquireLease(BlobClient blob, CancellationToken token) + public async Task AcquireLease(BlobClient blob) { try { - var leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), token); + var leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), CancellationToken.None); if (!leaseResult.IsSuccess) { _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}.", blob.Uri.AbsoluteUri); return AzureBlobLockResult.FailedLockResult(blob); } + else + { + _logger.LogInformation("AcquireLease: Lease was acquired for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); + } - var leaseId = leaseResult.LeaseId; - var lockResult = new AzureBlobLockResult(blob, true, leaseId, token); - var leaseRenewalCompletionSource = new TaskCompletionSource(); - + var leaseId = leaseResult.LeaseId; + var lockResult = new AzureBlobLockResult(blob, true, leaseId, CancellationToken.None); + var blob1 = lockResult.Blob; // Start a task that will renew the lease until the token is cancelled or the Release method is invoked _ = Task.Run(async () => { - try - { while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) { try { + var delay1 = TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds); await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); - await _blobLeaseService.RenewAsync(blob.Name, leaseId, lockResult.BlobOperationToken.Token); + if (! await blob1.ExistsAsync()) + { + break; + } + await _blobLeaseService.RenewAsync(blob.Name, leaseId, CancellationToken.None); _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", blob.Uri.AbsoluteUri, leaseId); @@ -94,14 +96,10 @@ public async Task AcquireLease(BlobClient blob, Cancellatio break; } } - } - finally - { - leaseRenewalCompletionSource.SetResult(true); - } + + }, lockResult.BlobOperationToken.Token); - lockResult.LeaseRenewalCompletionSource = leaseRenewalCompletionSource.Task; return lockResult; } catch (Exception ex) @@ -122,7 +120,6 @@ public async Task TryReleaseLockAsync(AzureBlobLockResult if (releaseResult) { _logger.LogInformation("ReleaseLockAsync: ReleaseLeaseStatus: {LeaseReleased} on the {BlobUri}.", true, releaseLock.Blob.Uri); - await releaseLock.LeaseRenewalCompletionSource; // Added return new AsyncOperationResult(true, null); } else diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs index 0bf6315396..cc139a44f2 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs @@ -3,7 +3,6 @@ using System; using System.Threading; -using System.Threading.Tasks; using Azure.Storage.Blobs; //using Microsoft.WindowsAzure.Storage.Blob; @@ -17,8 +16,6 @@ public class AzureBlobLockResult : IDisposable public BlobClient Blob { get; } - public Task LeaseRenewalCompletionSource { get; set; } - /// /// It will be cancelled when the renew task could not renew the lease. /// Operations can listen to this cancellation to stop execution once the lease could not be renewed. diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index c7c79509b3..65f6dae02f 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -163,7 +163,7 @@ public async Task TakeLockAsync(Uri blobUri, CancellationTo { return AzureBlobLockResult.FailedLockResult(blob); } - return await _blobLeaseManager.AcquireLease(blob, token); + return await _blobLeaseManager.AcquireLease(blob); } /// @@ -222,7 +222,7 @@ public async Task TryCleanAsync(AzureBlobLockResult blobLo // The operation will throw if the lease does not match bool deleteResult = await sourceBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: blobRequestConditions, - cancellationToken: token); + cancellationToken: CancellationToken.None); _logger.LogInformation("CleanAsync: Blob {Blob} was deleted {DeletedResult}. The leaseId: {LeaseId}", blobLock.Blob.Uri, deleteResult, blobLock.LeaseId); return new AsyncOperationResult(deleteResult, null); } diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index 7a6cd1f5fe..c128e65646 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -59,8 +59,8 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) var blobLeaseManager = new AzureBlobLeaseManager( serviceProvider.GetRequiredService>(), ValidateAzureBlobServiceClient(superstring), - _configuration.AzureContainerNameDestination, - superstring); + _configuration.AzureContainerNameSource, + ""); var source = new AzureStatsLogSource( @@ -70,6 +70,9 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) blobLeaseManager, serviceProvider.GetRequiredService>()); + superstring = _configuration.AzureAccountConnectionStringDestination; + superstring = superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + var dest = new AzureStatsLogDestination( ValidateAzureBlobServiceClient(superstring), _configuration.AzureContainerNameDestination, From 76dc6798710aab24fc1ec1713488897be07a3788 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 09:55:04 -0800 Subject: [PATCH 26/44] Polished BlobLeaseService.cs --- src/NuGet.Services.Storage/BlobLeaseService.cs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/NuGet.Services.Storage/BlobLeaseService.cs b/src/NuGet.Services.Storage/BlobLeaseService.cs index 7da733ceb2..ce7e4abad2 100644 --- a/src/NuGet.Services.Storage/BlobLeaseService.cs +++ b/src/NuGet.Services.Storage/BlobLeaseService.cs @@ -21,11 +21,10 @@ public class BlobLeaseService : IBlobLeaseService { private static readonly TimeSpan MinLeaseTime = TimeSpan.FromSeconds(15); private static readonly TimeSpan MaxLeaseTime = TimeSpan.FromSeconds(60); - private readonly ILogger _logger; private readonly BlobContainerClient _containerClient; private readonly string _basePath; - public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath, ILogger logger) + public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath) { if (blobServiceClient == null) { @@ -39,7 +38,6 @@ public BlobLeaseService(BlobServiceClient blobServiceClient, string containerNam { throw new ArgumentException("The base path must be provided.", nameof(basePath)); } - _logger = logger; _containerClient = blobServiceClient.GetBlobContainerClient(containerName); _basePath = string.IsNullOrEmpty(basePath) ? string.Empty : basePath.TrimEnd('/') + '/'; } @@ -47,22 +45,15 @@ public BlobLeaseService(BlobServiceClient blobServiceClient, string containerNam public async Task TryAcquireAsync(string resourceName, TimeSpan leaseTime, CancellationToken cancellationToken) { var blob = GetBlob(resourceName); - try { - var result = await TryAcquireAsync(blob, leaseTime, cancellationToken); - _logger.LogInformation("TryAcquireAsync: {resourceName} {leaseId} {leaseTime}", resourceName, result.LeaseId, leaseTime); - - return result; + return await TryAcquireAsync(blob, leaseTime, cancellationToken); ; } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) { // The lease file does not exist. Try to create it and lease it. - var result1 = await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); - _logger.LogInformation("TryAcquireAsync2: {resourceName} {leaseId}, {leaseTime}", resourceName, result1.LeaseId, leaseTime); - - return result1; + return await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); } } @@ -72,7 +63,6 @@ public async Task ReleaseAsync(string resourceName, string leaseId, Cancel { var blob = GetBlob(resourceName); var leaseClient = blob.GetBlobLeaseClient(leaseId); - _logger.LogInformation("ReleaseAsync: {resourceName} {leaseId}", resourceName, leaseId); await leaseClient.ReleaseAsync(conditions: null, cancellationToken: cancellationToken); @@ -103,8 +93,6 @@ public async Task RenewAsync(string resourceName, string leaseI try { - _logger.LogInformation("RenewAsync: {resourceName} {leaseId}", resourceName, leaseId); - var lease = await leaseClient.RenewAsync(conditions: null, cancellationToken: cancellationToken); return BlobLeaseResult.Success(lease); From f5c36eff8a480f68818f8b89d42adc997feb2dcb Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 10:03:02 -0800 Subject: [PATCH 27/44] Polished AzureBlobLockResult.cs --- .../AzureHelpers/AzureBlobLockResult.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs index cc139a44f2..07cfd7a624 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLockResult.cs @@ -4,7 +4,6 @@ using System; using System.Threading; using Azure.Storage.Blobs; -//using Microsoft.WindowsAzure.Storage.Blob; namespace Stats.AzureCdnLogs.Common { From a176ce0041127d424e52da1db697f89ef2ddb626 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 10:18:55 -0800 Subject: [PATCH 28/44] Polished AzureBlobLeaseManager.cs --- .../AzureHelpers/AzureBlobLeaseManager.cs | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index bf3f5d6574..1b2a32e0fa 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -9,8 +9,6 @@ using Stats.AzureCdnLogs.Common.Collect; using NuGet.Services.Storage; using Azure.Storage.Blobs.Specialized; -//using Microsoft.WindowsAzure.Storage; -//using Microsoft.WindowsAzure.Storage.Blob; namespace Stats.AzureCdnLogs.Common { @@ -23,7 +21,6 @@ public class AzureBlobLeaseManager public const int MaxRenewPeriodInSeconds = 60; // The lease will be renewed with a short interval before the the lease expires public const int OverlapRenewPeriodInSeconds = 20; - //private BlobRequestOptions _blobRequestOptions; private readonly ILogger _logger; private BlobLeaseService _blobLeaseService; @@ -38,19 +35,18 @@ public AzureBlobLeaseManager(ILogger logger, BlobServiceC else throw new ArgumentException(nameof(containerName)); } - - _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath, logger); + _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath); } /// /// Try to acquire a lease on the blob. If the acquire is successful the lease will be renewed at every 60 seconds. - /// In order to stop the renew task the needs to be invoked + /// In order to stop the renew task the needs to be invoked /// or the token to be cancelled. /// /// The blob to acquire the lease on. - /// A token to cancel the operation. - /// The renew task. - /// True if the lease was acquired. + /// An indicating the result of the lease acquisition. + /// If the lease is successfully acquired, the result will contain the lease ID and a cancellation token + /// source that can be used to stop the lease renewal task. public async Task AcquireLease(BlobClient blob) { try @@ -66,39 +62,38 @@ public async Task AcquireLease(BlobClient blob) _logger.LogInformation("AcquireLease: Lease was acquired for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); } - var leaseId = leaseResult.LeaseId; + var leaseId = leaseResult.LeaseId; var lockResult = new AzureBlobLockResult(blob, true, leaseId, CancellationToken.None); - var blob1 = lockResult.Blob; + var leasedBlob = lockResult.Blob; // Start a task that will renew the lease until the token is cancelled or the Release method is invoked _ = Task.Run(async () => { - while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) + while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) + { + try { - try - { - var delay1 = TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds); - await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); - if (! await blob1.ExistsAsync()) + await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); + if (!await leasedBlob.ExistsAsync()) { break; } await _blobLeaseService.RenewAsync(blob.Name, leaseId, CancellationToken.None); - _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", - blob.Uri.AbsoluteUri, - leaseId); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. LeaseId {LeaseId}.", - blob.Uri.AbsoluteUri, - leaseId); - lockResult.BlobOperationToken.Cancel(); - break; - } + _logger.LogInformation("RenewLeaseTask: Lease was renewed for BlobUri {BlobUri} and LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); } - - - + catch (Exception ex) + { + _logger.LogWarning(ex, "RenewLeaseTask: The Lease could not be renewed for BlobUri {BlobUri}. LeaseId {LeaseId}.", + blob.Uri.AbsoluteUri, + leaseId); + lockResult.BlobOperationToken.Cancel(); + break; + } + } + + + }, lockResult.BlobOperationToken.Token); return lockResult; } @@ -114,7 +109,7 @@ public async Task TryReleaseLockAsync(AzureBlobLockResult { try { - if(await releaseLock.Blob.ExistsAsync()) + if (await releaseLock.Blob.ExistsAsync()) { bool releaseResult = await _blobLeaseService.ReleaseAsync(releaseLock.Blob.Name, releaseLock.LeaseId, releaseLock.BlobOperationToken.Token); if (releaseResult) From 477bb5f202ce68f949151eecfce83e463b44926d Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 10:38:00 -0800 Subject: [PATCH 29/44] Polished Job.cs --- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 31 ++++++++++------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index c128e65646..f5a0745d31 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -11,7 +11,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -//using Microsoft.WindowsAzure.Storage; using NuGet.Jobs; using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; @@ -23,7 +22,7 @@ public class Job : JsonConfigurationJob { private const int DefaultExecutionTimeoutInSeconds = 14400; // 4 hours private const int MaxFilesToProcess = 4; - + private CollectAzureChinaCdnLogsConfiguration _configuration; private int _executionTimeoutInSeconds; private Collector _chinaCollector; @@ -40,41 +39,39 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) _configuration = serviceProvider.GetRequiredService>().Value; _executionTimeoutInSeconds = _configuration.ExecutionTimeoutInSeconds ?? DefaultExecutionTimeoutInSeconds; - var superstring = _configuration.AzureAccountConnectionStringSource; + var connectionStringSource = _configuration.AzureAccountConnectionStringSource; + var connectionStringDestination = _configuration.AzureAccountConnectionStringDestination; - if (string.IsNullOrEmpty(_configuration.AzureAccountConnectionStringSource)) + if (string.IsNullOrEmpty(connectionStringSource)) { - throw new ArgumentException(nameof(superstring)); + throw new ArgumentException(nameof(connectionStringSource)); } - if (string.IsNullOrEmpty(_configuration.AzureAccountConnectionStringDestination)) { - throw new ArgumentException(nameof(superstring)); + throw new ArgumentException(nameof(connectionStringDestination)); } - superstring = superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + connectionStringSource = connectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var blobLeaseManager = new AzureBlobLeaseManager( serviceProvider.GetRequiredService>(), - ValidateAzureBlobServiceClient(superstring), + ValidateAzureBlobServiceClient(connectionStringSource), _configuration.AzureContainerNameSource, ""); - var source = new AzureStatsLogSource( - ValidateAzureBlobServiceClient(superstring), + ValidateAzureBlobServiceClient(connectionStringSource), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / MaxFilesToProcess, blobLeaseManager, serviceProvider.GetRequiredService>()); - superstring = _configuration.AzureAccountConnectionStringDestination; - superstring = superstring.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + connectionStringDestination = connectionStringDestination.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var dest = new AzureStatsLogDestination( - ValidateAzureBlobServiceClient(superstring), + ValidateAzureBlobServiceClient(connectionStringDestination), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); @@ -93,7 +90,7 @@ public override async Task Run() // so we can't reliably terminate those if they get stuck or very slow. If migrated // to .NET 6+ then we can properly propagate the token and this hack would no longer // be needed. - + // Instead we'll wait a bit extra time after firing main CancellationTokenSource to // let it gracefully stop if it is indeed just slow, then stop the process and let it // retry processing on restart. @@ -124,13 +121,13 @@ public override async Task Run() if (aggregateExceptions != null) { - foreach(var ex in aggregateExceptions.InnerExceptions) + foreach (var ex in aggregateExceptions.InnerExceptions) { Logger.LogError(LogEvents.JobRunFailed, ex, ex.Message); } } - if(cts.IsCancellationRequested) + if (cts.IsCancellationRequested) { Logger.LogInformation("Execution exceeded the timeout of {ExecutionTimeoutInSeconds} seconds and it was cancelled.", _executionTimeoutInSeconds); } From 629cdcec14f355f0af0b2e100a0e4aef5ad3c6df Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 10:44:57 -0800 Subject: [PATCH 30/44] Fixed CDNLogsSanitizer scripts --- src/Stats.CDNLogsSanitizer/Job.cs | 15 ++++++++------- src/Stats.CDNLogsSanitizer/Processor.cs | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Stats.CDNLogsSanitizer/Job.cs b/src/Stats.CDNLogsSanitizer/Job.cs index 0518683efe..13a8d08b00 100644 --- a/src/Stats.CDNLogsSanitizer/Job.cs +++ b/src/Stats.CDNLogsSanitizer/Job.cs @@ -11,8 +11,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -//using Microsoft.WindowsAzure.Storage; -using Azure; using Azure.Storage.Blobs; using NuGet.Jobs; using Stats.AzureCdnLogs.Common; @@ -48,22 +46,25 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) var logHeaderDelimiter = _configuration.LogHeaderDelimiter ?? throw new ArgumentNullException(nameof(_configuration.LogHeaderDelimiter)); _logHeaderMetadata = new LogHeaderMetadata(logHeader, logHeaderDelimiter); _blobPrefix = _configuration.BlobPrefix ; - var superstring = _configuration.AzureAccountConnectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + + var connectionStringSource = _configuration.AzureAccountConnectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + var connectionStringDestination = _configuration.AzureAccountConnectionStringDestination.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + var blobLeaseManager = new AzureBlobLeaseManager( serviceProvider.GetRequiredService>(), - ValidateAzureCloudStorageAccount(superstring), + ValidateAzureCloudStorageAccount(connectionStringSource), _configuration.AzureContainerNameDestination, - superstring); + connectionStringSource); var source = new AzureStatsLogSource( - ValidateAzureCloudStorageAccount(superstring), + ValidateAzureCloudStorageAccount(connectionStringSource), _configuration.AzureContainerNameSource, _executionTimeoutInSeconds / _maxBlobsToProcess, blobLeaseManager, serviceProvider.GetRequiredService>()); var dest = new AzureStatsLogDestination( - ValidateAzureCloudStorageAccount(superstring), + ValidateAzureCloudStorageAccount(connectionStringDestination), _configuration.AzureContainerNameDestination, serviceProvider.GetRequiredService>()); diff --git a/src/Stats.CDNLogsSanitizer/Processor.cs b/src/Stats.CDNLogsSanitizer/Processor.cs index 6b15716a00..a0c5bd4472 100644 --- a/src/Stats.CDNLogsSanitizer/Processor.cs +++ b/src/Stats.CDNLogsSanitizer/Processor.cs @@ -9,7 +9,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -//using Microsoft.WindowsAzure.Storage; using Azure; using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; From 2403857bea39e5ae4954c520fe67c88486e0192f Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Wed, 29 Jan 2025 10:48:53 -0800 Subject: [PATCH 31/44] Polished tests --- .../Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs | 1 - tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs | 1 - tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs index c618ac31e9..f65df7d2d2 100644 --- a/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs +++ b/tests/Tests.Stats.AzureCdnLogs.Common/AzureBlobLockResultTests.cs @@ -3,7 +3,6 @@ using System; using System.Threading; -//using Microsoft.WindowsAzure.Storage.Blob; using Azure.Storage.Blobs; using Stats.AzureCdnLogs.Common; using Xunit; diff --git a/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs b/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs index e70073e3fa..4e57a61b39 100644 --- a/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs +++ b/tests/Tests.Stats.CDNLogsSanitizer/ProcessorTests.cs @@ -7,7 +7,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -//using Microsoft.WindowsAzure.Storage.Blob; using Azure.Storage.Blobs; using Azure; using Microsoft.Extensions.Logging; diff --git a/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs b/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs index 77491cf5bb..302264968e 100644 --- a/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs +++ b/tests/Tests.Stats.CollectAzureChinaCDNLogs/JobTests.cs @@ -6,7 +6,6 @@ using System.ComponentModel.Design; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -//using Microsoft.WindowsAzure.Storage; using Azure.Storage.Blobs; using Azure; using Moq; From 505096ad8a6af6bd19e9bf47979970d3b59346cd Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 11:22:35 -0800 Subject: [PATCH 32/44] Fixed extra semi-colon --- src/NuGet.Services.Storage/BlobLeaseService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NuGet.Services.Storage/BlobLeaseService.cs b/src/NuGet.Services.Storage/BlobLeaseService.cs index ce7e4abad2..6412f9d995 100644 --- a/src/NuGet.Services.Storage/BlobLeaseService.cs +++ b/src/NuGet.Services.Storage/BlobLeaseService.cs @@ -48,7 +48,7 @@ public async Task TryAcquireAsync(string resourceName, TimeSpan try { - return await TryAcquireAsync(blob, leaseTime, cancellationToken); ; + return await TryAcquireAsync(blob, leaseTime, cancellationToken); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) { From 6a13b42b78e87a4ba6cc3a072b303bd02af0c508 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 11:27:05 -0800 Subject: [PATCH 33/44] Moved throw to new line --- .../AzureHelpers/AzureBlobLeaseManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 1b2a32e0fa..1969238937 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -26,7 +26,8 @@ public class AzureBlobLeaseManager public AzureBlobLeaseManager(ILogger logger, BlobServiceClient blobServiceClient, string containerName, string basePath) { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _logger = logger ?? + throw new ArgumentNullException(nameof(logger)); if (blobServiceClient == null) throw new ArgumentNullException(nameof(blobServiceClient)); if (string.IsNullOrEmpty(containerName)) From 3651f313154ab2b82035b34aeea0034fa0ffa949 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 11:28:17 -0800 Subject: [PATCH 34/44] Moved throw to new line pt.2 --- .../AzureHelpers/AzureBlobLeaseManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 1969238937..5f04a20ea2 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -32,7 +32,8 @@ public AzureBlobLeaseManager(ILogger logger, BlobServiceC if (string.IsNullOrEmpty(containerName)) { - if (containerName == null) throw new ArgumentNullException(nameof(containerName)); + if (containerName == null) + throw new ArgumentNullException(nameof(containerName)); else throw new ArgumentException(nameof(containerName)); } From 47a50926ca67a9b16d0977c9efe5eb70137dd624 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 11:30:41 -0800 Subject: [PATCH 35/44] Changed to explicit type --- .../AzureHelpers/AzureBlobLeaseManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 5f04a20ea2..455458c8aa 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -53,7 +53,7 @@ public async Task AcquireLease(BlobClient blob) { try { - var leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), CancellationToken.None); + BlobLeaseResult leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), CancellationToken.None); if (!leaseResult.IsSuccess) { _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}.", blob.Uri.AbsoluteUri); From b778307f37f0ea9a3477368bc8b5888ea991d97a Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 11:36:23 -0800 Subject: [PATCH 36/44] Added more explicit types --- .../AzureHelpers/AzureBlobLeaseManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 455458c8aa..3955cbc35f 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -64,9 +64,9 @@ public async Task AcquireLease(BlobClient blob) _logger.LogInformation("AcquireLease: Lease was acquired for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); } - var leaseId = leaseResult.LeaseId; + string leaseId = leaseResult.LeaseId; var lockResult = new AzureBlobLockResult(blob, true, leaseId, CancellationToken.None); - var leasedBlob = lockResult.Blob; + BlobClient leasedBlob = lockResult.Blob; // Start a task that will renew the lease until the token is cancelled or the Release method is invoked _ = Task.Run(async () => { From 84c4699d0a047e6d55da20528747192ec5ed1345 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 12:55:51 -0800 Subject: [PATCH 37/44] Nit fixes --- src/NuGet.Services.Storage/BlobLeaseService.cs | 2 -- .../Collect/AzureStatsLogDestination.cs | 1 - .../Collect/AzureStatsLogSource.cs | 11 ++++++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/NuGet.Services.Storage/BlobLeaseService.cs b/src/NuGet.Services.Storage/BlobLeaseService.cs index 6412f9d995..04348180dc 100644 --- a/src/NuGet.Services.Storage/BlobLeaseService.cs +++ b/src/NuGet.Services.Storage/BlobLeaseService.cs @@ -63,9 +63,7 @@ public async Task ReleaseAsync(string resourceName, string leaseId, Cancel { var blob = GetBlob(resourceName); var leaseClient = blob.GetBlobLeaseClient(leaseId); - await leaseClient.ReleaseAsync(conditions: null, cancellationToken: cancellationToken); - return true; } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Conflict) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs index 0da25ccfeb..a58ab756ee 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs @@ -1,4 +1,3 @@ - // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index 65f6dae02f..11295924f0 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -49,7 +49,8 @@ public AzureStatsLogSource(BlobServiceClient blobServiceClient, if (string.IsNullOrEmpty(containerName)) { - if (containerName == null) throw new ArgumentNullException(nameof(containerName)); + if (containerName == null) + throw new ArgumentNullException(nameof(containerName)); else throw new ArgumentException(nameof(containerName)); } _container = _blobServiceClient.GetBlobContainerClient(containerName); @@ -88,7 +89,7 @@ public async Task> GetFilesAsync(int maxResults, CancellationTo _logger.LogInformation("Found blob {BlobName}, determining lease status...", blobItem.Name); - var blob = _container.GetBlobClient(blobItem.Name); + BlobClient blob = _container.GetBlobClient(blobItem.Name); BlobProperties properties = await blob.GetPropertiesAsync(cancellationToken: token); if (properties.LeaseStatus != LeaseStatus.Unlocked) { @@ -240,9 +241,9 @@ private async Task GetBlobAsync(Uri blobUri) { try { - var blobClient = new BlobClient(blobUri); - var properties = await blobClient.GetPropertiesAsync(); - return blobClient; + var _blobClient = new BlobClient(blobUri); + var properties = await _blobClient.GetPropertiesAsync(); + return _blobClient; } catch (Exception) { From b7d9f1c4ab6cabb114f68ad7a3f6bb80e57a9105 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 15:20:42 -0800 Subject: [PATCH 38/44] Added extra comments --- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index f5a0745d31..b3f7429736 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -53,6 +53,7 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) throw new ArgumentException(nameof(connectionStringDestination)); } + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 connectionStringSource = connectionStringSource.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var blobLeaseManager = new AzureBlobLeaseManager( @@ -68,6 +69,7 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) blobLeaseManager, serviceProvider.GetRequiredService>()); + // workaround for https://github.com/Azure/azure-sdk-for-net/issues/44373 connectionStringDestination = connectionStringDestination.Replace("SharedAccessSignature=?", "SharedAccessSignature="); var dest = new AzureStatsLogDestination( From c5ac84ad47b88d6f6161b09be1224cd487788c59 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Fri, 31 Jan 2025 15:38:49 -0800 Subject: [PATCH 39/44] Added non-negative waiting time check --- .../AzureHelpers/AzureBlobLeaseManager.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 3955cbc35f..e75411668c 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -70,11 +70,14 @@ public async Task AcquireLease(BlobClient blob) // Start a task that will renew the lease until the token is cancelled or the Release method is invoked _ = Task.Run(async () => { + + int sleepBeforeRenewInSeconds = MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds < 0 ? MaxRenewPeriodInSeconds : MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds; + while (!lockResult.BlobOperationToken.Token.IsCancellationRequested) { try { - await Task.Delay(TimeSpan.FromSeconds(MaxRenewPeriodInSeconds - OverlapRenewPeriodInSeconds), lockResult.BlobOperationToken.Token); + await Task.Delay(TimeSpan.FromSeconds(sleepBeforeRenewInSeconds), lockResult.BlobOperationToken.Token); if (!await leasedBlob.ExistsAsync()) { break; From d2d91f9c7f52f321ba251befda98564259cf5175 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 3 Feb 2025 10:03:26 -0800 Subject: [PATCH 40/44] Fixed token --- src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs index 11295924f0..93726eb2c2 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs @@ -223,7 +223,7 @@ public async Task TryCleanAsync(AzureBlobLockResult blobLo // The operation will throw if the lease does not match bool deleteResult = await sourceBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, conditions: blobRequestConditions, - cancellationToken: CancellationToken.None); + cancellationToken: token); _logger.LogInformation("CleanAsync: Blob {Blob} was deleted {DeletedResult}. The leaseId: {LeaseId}", blobLock.Blob.Uri, deleteResult, blobLock.LeaseId); return new AsyncOperationResult(deleteResult, null); } From 24988cc5cf99f884fc2a45ddcb13fa7f0c05ffe2 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 3 Feb 2025 10:16:20 -0800 Subject: [PATCH 41/44] Added a new check --- .../Collect/AzureStatsLogDestination.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs index a58ab756ee..a6a48a7113 100644 --- a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs +++ b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs @@ -89,9 +89,13 @@ public async Task TryWriteAsync(Stream inputStream, Action writeAction(inputStream, resultStream); } - await resultStream.FlushAsync(); - _logger.LogInformation("WriteAsync: End write to {DestinationFileName}", destinationFileName); - return new AsyncOperationResult(true, null); + if(!(await blobClient.ExistsAsync(token))) + { + await resultStream.FlushAsync(); + _logger.LogInformation("WriteAsync: End write to {DestinationFileName}", destinationFileName); + return new AsyncOperationResult(true, null); + } + } _logger.LogInformation("WriteAsync: The destination file {DestinationFileName}, was already present.", destinationFileName); return new AsyncOperationResult(false, null); From 2dd139ca7fc14b981fa57eba9c1574b94a6201f8 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 3 Feb 2025 10:31:43 -0800 Subject: [PATCH 42/44] Removed unneccessary cancellation token --- .../AzureHelpers/AzureBlobLeaseManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index e75411668c..34070cd6ed 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -77,7 +77,7 @@ public async Task AcquireLease(BlobClient blob) { try { - await Task.Delay(TimeSpan.FromSeconds(sleepBeforeRenewInSeconds), lockResult.BlobOperationToken.Token); + await Task.Delay(TimeSpan.FromSeconds(sleepBeforeRenewInSeconds)); if (!await leasedBlob.ExistsAsync()) { break; From 227626f17f5319754e40cc20fb6740a9e7d53b10 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Mon, 3 Feb 2025 11:15:04 -0800 Subject: [PATCH 43/44] Added flag --- src/NuGet.Services.Storage/BlobLeaseService.cs | 10 ++++++++-- .../AzureHelpers/AzureBlobLeaseManager.cs | 7 ++++--- src/Stats.CollectAzureChinaCDNLogs/Job.cs | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/NuGet.Services.Storage/BlobLeaseService.cs b/src/NuGet.Services.Storage/BlobLeaseService.cs index 04348180dc..3b73ba98ef 100644 --- a/src/NuGet.Services.Storage/BlobLeaseService.cs +++ b/src/NuGet.Services.Storage/BlobLeaseService.cs @@ -23,8 +23,9 @@ public class BlobLeaseService : IBlobLeaseService private static readonly TimeSpan MaxLeaseTime = TimeSpan.FromSeconds(60); private readonly BlobContainerClient _containerClient; private readonly string _basePath; + private readonly bool _createBlobWhenMissing; - public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath) + public BlobLeaseService(BlobServiceClient blobServiceClient, string containerName, string basePath, Boolean createBlobsWhenMissing = true) { if (blobServiceClient == null) { @@ -40,6 +41,7 @@ public BlobLeaseService(BlobServiceClient blobServiceClient, string containerNam } _containerClient = blobServiceClient.GetBlobContainerClient(containerName); _basePath = string.IsNullOrEmpty(basePath) ? string.Empty : basePath.TrimEnd('/') + '/'; + _createBlobWhenMissing = createBlobsWhenMissing; } public async Task TryAcquireAsync(string resourceName, TimeSpan leaseTime, CancellationToken cancellationToken) @@ -53,7 +55,11 @@ public async Task TryAcquireAsync(string resourceName, TimeSpan catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) { // The lease file does not exist. Try to create it and lease it. - return await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); + if (_createBlobWhenMissing) + { + return await TryCreateAndAcquireAsync(blob, leaseTime, cancellationToken); + } + return BlobLeaseResult.Failure(); } } diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index 34070cd6ed..b6cfe1c115 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -23,8 +23,9 @@ public class AzureBlobLeaseManager public const int OverlapRenewPeriodInSeconds = 20; private readonly ILogger _logger; private BlobLeaseService _blobLeaseService; + private Boolean _createBlobWhenNotFound; - public AzureBlobLeaseManager(ILogger logger, BlobServiceClient blobServiceClient, string containerName, string basePath) + public AzureBlobLeaseManager(ILogger logger, BlobServiceClient blobServiceClient, string containerName, string basePath, bool createBlobWhenNotFound = true) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -36,8 +37,8 @@ public AzureBlobLeaseManager(ILogger logger, BlobServiceC throw new ArgumentNullException(nameof(containerName)); else throw new ArgumentException(nameof(containerName)); } - - _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath); + _createBlobWhenNotFound = createBlobWhenNotFound; + _blobLeaseService = new BlobLeaseService(blobServiceClient, containerName, basePath, _createBlobWhenNotFound); } /// diff --git a/src/Stats.CollectAzureChinaCDNLogs/Job.cs b/src/Stats.CollectAzureChinaCDNLogs/Job.cs index b3f7429736..61f1bbde91 100644 --- a/src/Stats.CollectAzureChinaCDNLogs/Job.cs +++ b/src/Stats.CollectAzureChinaCDNLogs/Job.cs @@ -60,7 +60,8 @@ public void InitializeJobConfiguration(IServiceProvider serviceProvider) serviceProvider.GetRequiredService>(), ValidateAzureBlobServiceClient(connectionStringSource), _configuration.AzureContainerNameSource, - ""); + "", + false); var source = new AzureStatsLogSource( ValidateAzureBlobServiceClient(connectionStringSource), From d4e996cc7fcd66aa0c82da64a8dff2268a66b401 Mon Sep 17 00:00:00 2001 From: Lanaparezanin Date: Thu, 6 Feb 2025 15:43:30 -0800 Subject: [PATCH 44/44] Fixed a nit --- .../AzureHelpers/AzureBlobLeaseManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs index b6cfe1c115..aaf52d0328 100644 --- a/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs +++ b/src/Stats.AzureCdnLogs.Common/AzureHelpers/AzureBlobLeaseManager.cs @@ -55,14 +55,14 @@ public async Task AcquireLease(BlobClient blob) try { BlobLeaseResult leaseResult = await _blobLeaseService.TryAcquireAsync(blob.Name, TimeSpan.FromSeconds(MaxRenewPeriodInSeconds), CancellationToken.None); - if (!leaseResult.IsSuccess) + if (leaseResult.IsSuccess) { - _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}.", blob.Uri.AbsoluteUri); - return AzureBlobLockResult.FailedLockResult(blob); + _logger.LogInformation("AcquireLease: Lease was acquired for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); } else { - _logger.LogInformation("AcquireLease: Lease was acquired for BlobUri {BlobUri}.", blob.Uri.AbsoluteUri); + _logger.LogInformation("AcquireLease: The operation was cancelled or the blob lease is already taken. Blob {BlobUri}.", blob.Uri.AbsoluteUri); + return AzureBlobLockResult.FailedLockResult(blob); } string leaseId = leaseResult.LeaseId;