Skip to content

Commit

Permalink
Cert rotator (#4617)
Browse files Browse the repository at this point in the history
* WIP

* Add certificate handling for auth

* Add cert handling for upload.py

* Remove certdeploy

* Remove old constants

* Add CertHelper code and build step

* Move build step for CertHelper

* Address PR feedback

* Remove islinux

* Add CertRotator to CertHelper

* WIP

* Fix improper list() usage

* Yaml testing changes

* Make RunCommand verbose

* Testing

* Add feature to RunCommand to not echo stdout

* Remove pragma and fixup tests

* Update moq version

* Remove testing changes

* Remove a few more testing changes

* Remove last testing change

* Add simple logging

* Address pr feedback

* Address PR feedback
  • Loading branch information
DrewScoggins authored Jan 21, 2025
1 parent 0db4c9b commit 7b8c420
Show file tree
Hide file tree
Showing 15 changed files with 696 additions and 21 deletions.
15 changes: 14 additions & 1 deletion scripts/performance/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import os
import sys
import time
import base64
from typing import Callable, List, Optional, Tuple, Type, TypeVar


Expand Down Expand Up @@ -138,6 +139,10 @@ def get_packages_directory() -> str:
'''
return os.path.join(get_artifacts_directory(), 'packages')

def base64_to_bytes(base64_string: str) -> bytes:
byte_data = base64.b64decode(base64_string)
return byte_data

@contextmanager
def push_dir(path: Optional[str] = None):
'''
Expand Down Expand Up @@ -233,6 +238,7 @@ def __init__(
cmdline: List[str],
success_exit_codes: Optional[List[int]] = None,
verbose: bool = False,
echo: bool = True,
retry: int = 0):
if cmdline is None:
raise TypeError('Unspecified command line to be executed.')
Expand All @@ -242,6 +248,7 @@ def __init__(
self.__cmdline = cmdline
self.__verbose = verbose
self.__retry = retry
self.__echo = echo

if success_exit_codes is None:
self.__success_exit_codes = [0]
Expand All @@ -261,6 +268,11 @@ def success_exit_codes(self) -> List[int]:
'''
return self.__success_exit_codes

@property
def echo(self) -> bool:
'''Enables/Disables echoing of STDOUT'''
return self.__echo

@property
def verbose(self) -> bool:
'''Enables/Disables verbosity.'''
Expand Down Expand Up @@ -296,7 +308,8 @@ def __runinternal(self, working_directory: Optional[str] = None) -> Tuple[int, s
line = raw_line.decode('utf-8', errors='backslashreplace')
self.__stdout.write(line)
line = line.rstrip()
getLogger().info(line)
if self.echo:
getLogger().info(line)
proc.wait()
return (proc.returncode, quoted_cmdline)

Expand Down
3 changes: 2 additions & 1 deletion scripts/performance/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
UPLOAD_STORAGE_URI = 'https://pvscmdupload.{}.core.windows.net'
UPLOAD_QUEUE = 'resultsqueue'
TENANT_ID = '72f988bf-86f1-41af-91ab-2d7cd011db47'
CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
ARC_CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
CERT_CLIENT_ID = '8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc'
37 changes: 26 additions & 11 deletions scripts/run_performance_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,32 @@ def run_performance_job(args: RunPerformanceJobArgs):
getLogger().info("Copying global.json to payload directory")
shutil.copy(os.path.join(args.performance_repo_dir, 'global.json'), os.path.join(performance_payload_dir, 'global.json'))

# Building CertHelper needs to happen here as we need it on every run. This also means that we will need to move the calculation
# of the parameters needed outside of the if block

framework = os.environ["PERFLAB_Framework"]
os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
if args.os_group == "windows":
runtime_id = f"win-{args.architecture}"
elif args.os_group == "osx":
runtime_id = f"osx-{args.architecture}"
else:
runtime_id = f"linux-{args.architecture}"

dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")

RunCommand([
dotnet_executable_path, "publish",
"-c", "Release",
"-o", os.path.join(payload_dir, "certhelper"),
"-f", framework,
"-r", runtime_id,
"--self-contained",
os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"),
f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}",
"-p:DisableTransitiveFrameworkReferenceDownloads=true"],
verbose=True).run()

if args.is_scenario:
set_environment_variable("DOTNET_ROOT", ci_setup_arguments.install_dir, save_to_pipeline=True)
getLogger().info(f"Set DOTNET_ROOT to {ci_setup_arguments.install_dir}")
Expand All @@ -782,17 +808,6 @@ def run_performance_job(args: RunPerformanceJobArgs):
set_environment_variable("PATH", new_path, save_to_pipeline=True)
getLogger().info(f"Set PATH to {new_path}")

framework = os.environ["PERFLAB_Framework"]
os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
if args.os_group == "windows":
runtime_id = f"win-{args.architecture}"
elif args.os_group == "osx":
runtime_id = f"osx-{args.architecture}"
else:
runtime_id = f"linux-{args.architecture}"

dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")

os.environ["MSBUILDDISABLENODEREUSE"] = "1" # without this, MSbuild will be kept alive

# build Startup
Expand Down
27 changes: 19 additions & 8 deletions scripts/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
from azure.storage.blob import BlobClient, ContentSettings
from azure.storage.queue import QueueClient, TextBase64EncodePolicy
from azure.core.exceptions import ResourceExistsError, ClientAuthenticationError
from azure.identity import DefaultAzureCredential, ClientAssertionCredential
from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential
from traceback import format_exc
from glob import glob
from performance.common import retry_on_exception
from performance.constants import TENANT_ID, CLIENT_ID
from performance.common import retry_on_exception, RunCommand, helixpayload, base64_to_bytes, extension
from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_CLIENT_ID
import os
import json

Expand All @@ -32,14 +32,25 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag
credential = None
try:
dac = DefaultAzureCredential()
credential = ClientAssertionCredential(TENANT_ID, CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
credential = ClientAssertionCredential(TENANT_ID, ARC_CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
credential.get_token("https://storage.azure.com/.default")
except ClientAuthenticationError as ex:
getLogger().info("Unable to use managed identity. Falling back to environment variable.")
credential = os.getenv(sas_token_env)
credential = None
getLogger().info("Unable to use managed identity. Falling back to certificate.")
cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]
cert_helper = RunCommand(cmd_line, None, True, False, 0)
cert_helper.run()
for cert in cert_helper.stdout.splitlines():
credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert))
try:
credential.get_token("https://storage.azure.com/.default")
except ClientAuthenticationError as ex:
credential = None
continue
if credential is None:
getLogger().error("Sas token environment variable {} was not defined.".format(sas_token_env))
return 1
getLogger().error("Unable to authenticate with managed identity or certificates.")
getLogger().info("Falling back to environment variable.")
credential = os.getenv(sas_token_env)

files = glob(globpath, recursive=True)
any_upload_or_queue_failed = False
Expand Down
12 changes: 12 additions & 0 deletions src/tools/CertHelper/AssemblyInfo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;

[assembly: InternalsVisibleTo("CertHelperTests")]
namespace CertHelper;
internal class AssemblyInfo
{
}
20 changes: 20 additions & 0 deletions src/tools/CertHelper/CertHelper.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFramework>
<!-- Supported target frameworks -->
<TargetFramework Condition="'$(TargetFramework)' == ''">net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Core" Version="1.44.1" />
<PackageReference Include="Azure.Identity" Version="1.11.4" />
<PackageReference Include="Azure.Security.KeyVault.Certificates" Version="4.7.0" />
<PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.7.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
</ItemGroup>

</Project>
28 changes: 28 additions & 0 deletions src/tools/CertHelper/CertHelper.sln
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.12.35514.174 d17.12
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertHelper", "CertHelper.csproj", "{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertRotatorTests", "..\CertHelperTests\CertRotatorTests.csproj", "{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.Build.0 = Release|Any CPU
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
14 changes: 14 additions & 0 deletions src/tools/CertHelper/Constants.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace CertHelper;
public class Constants
{
public static readonly string Cert1Name = "LabCert1";
public static readonly string Cert2Name = "LabCert2";
public static readonly Uri Cert1Id = new Uri("https://test.vault.azure.net/certificates/LabCert1/07a7d98bf4884e5c40e690e02b96b3b4");
public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c41e690e02b96b3b4");
}
35 changes: 35 additions & 0 deletions src/tools/CertHelper/IX509Store.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;

namespace CertHelper;
public interface IX509Store
{
X509Certificate2Collection Certificates { get; }
string? Name { get; }
StoreLocation Location { get; }
X509Store GetX509Store();
}

public class TestableX509Store : IX509Store
{
public X509Certificate2Collection Certificates { get => store.Certificates; }

public string? Name => store.Name;

public StoreLocation Location => store.Location;

private X509Store store;
public TestableX509Store(OpenFlags flags = OpenFlags.ReadOnly)
{
store = new X509Store(StoreName.My, StoreLocation.CurrentUser, flags);
}

public X509Store GetX509Store()
{
return store;
}
}
110 changes: 110 additions & 0 deletions src/tools/CertHelper/KeyVaultCert.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
using Azure;
using Azure.Core;
using Azure.Identity;
using Azure.Security.KeyVault.Certificates;
using Azure.Security.KeyVault.Secrets;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;

namespace CertHelper;

public class KeyVaultCert
{
private readonly string _keyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/";
private readonly string _tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47";
private readonly string _clientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc";

public X509Certificate2Collection KeyVaultCertificates { get; set; }
public ILocalCert LocalCerts { get; set; }
private TokenCredential _credential { get; set; }
private CertificateClient _certClient { get; set; }
private SecretClient _secretClient { get; set; }

public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient = null, SecretClient? secretClient = null, ILocalCert? localCerts = null)
{
LocalCerts = localCerts ?? new LocalCert();
_credential = cred ?? GetCertifcateCredentialAsync(_tenantId, _clientId, LocalCerts.Certificates).Result;
_certClient = certClient ?? new CertificateClient(new Uri(_keyVaultUrl), _credential);
_secretClient = secretClient ?? new SecretClient(new Uri(_keyVaultUrl), _credential);
KeyVaultCertificates = new X509Certificate2Collection();
}

public async Task LoadKeyVaultCertsAsync()
{
KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert1Name));
KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert2Name));

if (KeyVaultCertificates.Where(c => c == null).Count() > 0)
{
throw new Exception("One or more certificates not found");
}
}

private async Task<ClientCertificateCredential> GetCertifcateCredentialAsync(string tenantId, string clientId, X509Certificate2Collection certCollection)
{
ClientCertificateCredential? ccc = null;
Exception? exception = null;
foreach (var cert in certCollection)
{
try
{
ccc = new ClientCertificateCredential(tenantId, clientId, cert);
await ccc.GetTokenAsync(new TokenRequestContext(new string[] { "https://vault.azure.net/.default" }));
break;
}
catch (Exception ex)
{
ccc = null;
exception = ex;
}
}
if(ccc == null)
{
throw new Exception("Both certificates failed to authenticate", exception);
}
return ccc;
}

private async Task<X509Certificate2> FindCertificateInKeyVaultAsync(string certName)
{
var keyVaultCert = await _certClient.GetCertificateAsync(certName);
if(keyVaultCert.Value == null)
{
throw new Exception("Certificate not found in Key Vault");
}
var secret = await _secretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last());
if(secret.Value == null)
{
throw new Exception("Certificate secret not found in Key Vault");
}
var certBytes = Convert.FromBase64String(secret.Value.Value);
#if NET9_0_OR_GREATER
var cert = X509CertificateLoader.LoadPkcs12(certBytes, "", X509KeyStorageFlags.Exportable);
#else
var cert = new X509Certificate2(certBytes, "", X509KeyStorageFlags.Exportable);
#endif
return cert;
}

public bool ShouldRotateCerts()
{
var keyVaultThumbprints = new HashSet<string>();
foreach (var cert in KeyVaultCertificates)
{
keyVaultThumbprints.Add(cert.Thumbprint);
}
foreach(var cert in LocalCerts.Certificates)
{
if (!keyVaultThumbprints.Contains(cert.Thumbprint))
{
return true;
}
}
return false;
}
}
Loading

0 comments on commit 7b8c420

Please sign in to comment.