diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 234db6d1..b13ca0ac 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0.200-focal +FROM mcr.microsoft.com/dotnet/sdk:6.0.300-focal # Installing mono makes `dotnet test` work without errors even for net472. # But installing it takes a long time, so it's excluded by default. diff --git a/.gitignore b/.gitignore index 65f94776..3bb49919 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,7 @@ _TeamCity* # Visual Studio code coverage results *.coverage *.coveragexml +/coveragereport/ # NCrunch _NCrunch_* diff --git a/Apply-Template.ps1 b/Apply-Template.ps1 deleted file mode 100644 index d659670b..00000000 --- a/Apply-Template.ps1 +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env pwsh - -<# -.SYNOPSIS -Applies the template to another repo in a semi-destructive way. -Always apply to a clean working copy so that undesired updates can be easily reverted. -.PARAMETER Path -The path to the root of the repo to be updated with the latest version of this template. -#> - -[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Medium')] -Param( - [Parameter(Mandatory=$true)] - [ValidateScript({Test-Path $_})] - [string]$Path -) - -Push-Location $Path -try { - # Look for our own initial commit in the target repo's history. - # If it's there, they've already switched to using git merge to freshen up. - # Using Apply-Template would just complicate future merges, so block it. - git log 05f49ce799c1f9cc696d53eea89699d80f59f833 ^HEAD | Out-Null - if ($LASTEXITCODE -eq 0) { - Write-Error 'The target repo already has Library.Template history merged into it. Use `git merge` instead of this script to freshen your repo. See the README.md file for details.' - exit 1 - } -} finally { - Pop-Location -} - -Write-Host "Updating $Path" -robocopy /mir $PSScriptRoot/azure-pipelines $Path/azure-pipelines -robocopy /mir $PSScriptRoot/.devcontainer $Path/.devcontainer -robocopy /mir $PSScriptRoot/.github $Path/.github -robocopy /mir $PSScriptRoot/.vscode $Path/.vscode -robocopy /mir $PSScriptRoot/tools $Path/tools -robocopy $PSScriptRoot $Path Directory.Build.* global.json init.* azure-pipelines.yml .gitignore .gitattributes .editorconfig -robocopy $PSScriptRoot/src $Path/src Directory.Build.* .editorconfig AssemblyInfo.cs -robocopy $PSScriptRoot/test $Path/test Directory.Build.* .editorconfig -Remove-Item $Path/azure-pipelines/expand-template.yml diff --git a/Directory.Build.props b/Directory.Build.props index de17866f..29ab3472 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -43,7 +43,7 @@ - + @@ -80,7 +80,7 @@ It's also not necessary to generate these assets. --> - <_WpfTempProjectNuGetFilePathNoExt>$(RepoRootPath)obj\$(_TargetAssemblyProjectName)\$(_TargetAssemblyProjectName)$(MSBuildProjectExtension).nuget.g + <_WpfTempProjectNuGetFilePathNoExt>$(BaseIntermediateOutputPath)..\$(_TargetAssemblyProjectName)\$(_TargetAssemblyProjectName)$(MSBuildProjectExtension).nuget.g false false diff --git a/SUPPORT.md b/SUPPORT.md index 4af9192f..6f77c1e9 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,13 +1,13 @@ -# Support - -## How to file issues and get help - -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or -feature request as a new Issue. - -For help and questions about using this project, please file a GitHub issue. - -## Microsoft Support Policy - -Support for this C# projection generator is limited to the resources listed above. +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please file a GitHub issue. + +## Microsoft Support Policy + +Support for this C# projection generator is limited to the resources listed above. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed3e23ed..8a2842d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,15 +17,20 @@ parameters: displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines +- name: RunTests + displayName: Run tests + type: boolean + default: true variables: TreatWarningsAsErrors: true DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release # #codecov_token: # Get a new one from https://codecov.io/ - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ jobs: - template: azure-pipelines/build.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} + RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/Convert-PDB.ps1 b/azure-pipelines/Convert-PDB.ps1 new file mode 100644 index 00000000..f119a164 --- /dev/null +++ b/azure-pipelines/Convert-PDB.ps1 @@ -0,0 +1,42 @@ +<# +.SYNOPSIS + Converts between Windows PDB and Portable PDB formats. +.PARAMETER DllPath + The path to the DLL whose PDB is to be converted. +.PARAMETER PdbPath + The path to the PDB to convert. May be omitted if the DLL was compiled on this machine and the PDB is still at its original path. +.PARAMETER OutputPath + The path of the output PDB to write. +#> +[CmdletBinding()] +Param( + [Parameter(Mandatory=$true,Position=0)] + [string]$DllPath, + [Parameter()] + [string]$PdbPath, + [Parameter(Mandatory=$true,Position=1)] + [string]$OutputPath +) + +if ($IsMacOS -or $IsLinux) { + Write-Error "This script only works on Windows" + return +} + +$version = '1.1.0-beta2-21101-01' +$baseDir = "$PSScriptRoot/../obj/tools" +$pdb2pdbpath = "$baseDir/Microsoft.DiaSymReader.Pdb2Pdb.$version/tools/Pdb2Pdb.exe" +if (-not (Test-Path $pdb2pdbpath)) { + if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } + $baseDir = (Resolve-Path $baseDir).Path # Normalize it + Write-Verbose "& (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null" + & (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null +} + +$args = $DllPath,'/out',$OutputPath,'/nowarn','0021' +if ($PdbPath) { + $args += '/pdb',$PdbPath +} + +Write-Verbose "$pdb2pdbpath $args" +& $pdb2pdbpath $args diff --git a/azure-pipelines/Get-CodeCovTool.ps1 b/azure-pipelines/Get-CodeCovTool.ps1 new file mode 100644 index 00000000..ca580b4d --- /dev/null +++ b/azure-pipelines/Get-CodeCovTool.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS + Downloads the CodeCov.io uploader tool and returns the path to it. +.PARAMETER AllowSkipVerify + Allows skipping signature verification of the downloaded tool if gpg is not installed. +#> +[CmdletBinding()] +Param( + [switch]$AllowSkipVerify +) + +if ($IsMacOS) { + $codeCovUrl = "https://uploader.codecov.io/latest/macos/codecov" + $toolName = 'codecov' +} +elseif ($IsLinux) { + $codeCovUrl = "https://uploader.codecov.io/latest/linux/codecov" + $toolName = 'codecov' +} +else { + $codeCovUrl = "https://uploader.codecov.io/latest/windows/codecov.exe" + $toolName = 'codecov.exe' +} + +$shaSuffix = ".SHA256SUM" +$sigSuffix = $shaSuffix + ".sig" + +Function Get-FileFromWeb([Uri]$Uri, $OutDir) { + $OutFile = Join-Path $OutDir $Uri.Segments[-1] + if (!(Test-Path $OutFile)) { + Write-Verbose "Downloading $Uri..." + if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir | Out-Null } + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) + } finally { + # This try/finally causes the script to abort + } + } + + $OutFile +} + +$toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" +$binaryToolsPath = Join-Path $toolsPath codecov +$testingPath = Join-Path $binaryToolsPath unverified +$finalToolPath = Join-Path $binaryToolsPath $toolName + +if (!(Test-Path $finalToolPath)) { + if (Test-Path $testingPath) { + Remove-Item -Recurse -Force $testingPath # ensure we download all matching files + } + $tool = Get-FileFromWeb $codeCovUrl $testingPath + $sha = Get-FileFromWeb "$codeCovUrl$shaSuffix" $testingPath + $sig = Get-FileFromWeb "$codeCovUrl$sigSuffix" $testingPath + $key = Get-FileFromWeb https://keybase.io/codecovsecurity/pgp_keys.asc $testingPath + + if ((Get-Command gpg -ErrorAction SilentlyContinue)) { + Write-Host "Importing codecov key" -ForegroundColor Yellow + gpg --import $key + Write-Host "Verifying signature on codecov hash" -ForegroundColor Yellow + gpg --verify $sig $sha + } else { + if ($AllowSkipVerify) { + Write-Warning "gpg not found. Unable to verify hash signature." + } else { + throw "gpg not found. Unable to verify hash signature. Install gpg or add -AllowSkipVerify to override." + } + } + + Write-Host "Verifying hash on downloaded tool" -ForegroundColor Yellow + $actualHash = (Get-FileHash -Path $tool -Algorithm SHA256).Hash + $expectedHash = (Get-Content $sha).Split()[0] + if ($actualHash -ne $expectedHash) { + # Validation failed. Delete the tool so we can't execute it. + #Remove-Item $codeCovPath + throw "codecov uploader tool failed signature validation." + } + + Copy-Item $tool $finalToolPath + + if ($IsMacOS -or $IsLinux) { + chmod u+x $finalToolPath + } +} + +return $finalToolPath diff --git a/azure-pipelines/Get-SymbolFiles.ps1 b/azure-pipelines/Get-SymbolFiles.ps1 index fccb1bb1..9183c340 100644 --- a/azure-pipelines/Get-SymbolFiles.ps1 +++ b/azure-pipelines/Get-SymbolFiles.ps1 @@ -49,8 +49,13 @@ $PDBs |% { $BinaryImagePath = $dllPath } elseif (Test-Path $exePath) { $BinaryImagePath = $exePath + } else { + Write-Warning "`"$_`" found with no matching binary file." + $BinaryImagePath = $null } - Write-Output $BinaryImagePath - Write-Output $_.FullName + if ($BinaryImagePath) { + Write-Output $BinaryImagePath + Write-Output $_.FullName + } } diff --git a/azure-pipelines/Install-NuGetPackage.ps1 b/azure-pipelines/Install-NuGetPackage.ps1 index 0bf05710..13967233 100644 --- a/azure-pipelines/Install-NuGetPackage.ps1 +++ b/azure-pipelines/Install-NuGetPackage.ps1 @@ -45,6 +45,8 @@ try { $p = Start-Process $nugetPath $nugetArgs -NoNewWindow -Wait -PassThru if ($p.ExitCode -ne 0) { throw } } + + Write-Output (Get-ChildItem "$PackagesDir\$PackageId.*")[0].FullName } finally { Pop-Location } diff --git a/azure-pipelines/Merge-CodeCoverage.ps1 b/azure-pipelines/Merge-CodeCoverage.ps1 new file mode 100644 index 00000000..91ab67ab --- /dev/null +++ b/azure-pipelines/Merge-CodeCoverage.ps1 @@ -0,0 +1,46 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Merges code coverage reports. +.PARAMETER Path + The path(s) to search for Cobertura code coverage reports. +.PARAMETER Format + The format for the merged result. The default is Cobertura +.PARAMETER OutputDir + The directory the merged result will be written to. The default is `coveragereport` in the root of this repo. +#> +[CmdletBinding()] +Param( + [Parameter(Mandatory=$true)] + [string[]]$Path, + [ValidateSet('Badges', 'Clover', 'Cobertura', 'CsvSummary', 'Html', 'Html_Dark', 'Html_Light', 'HtmlChart', 'HtmlInline', 'HtmlInline_AzurePipelines', 'HtmlInline_AzurePipelines_Dark', 'HtmlInline_AzurePipelines_Light', 'HtmlSummary', 'JsonSummary', 'Latex', 'LatexSummary', 'lcov', 'MarkdownSummary', 'MHtml', 'PngChart', 'SonarQube', 'TeamCitySummary', 'TextSummary', 'Xml', 'XmlSummary')] + [string]$Format='Cobertura', + [string]$OutputDir=("$PSScriptRoot/../coveragereport") +) + +$RepoRoot = [string](Resolve-Path $PSScriptRoot/..) + +if (!(Test-Path $RepoRoot/obj/reportgenerator*)) { + dotnet tool install --tool-path $RepoRoot/obj dotnet-reportgenerator-globaltool --version 5.1.9 --configfile $PSScriptRoot/justnugetorg.nuget.config +} + +Write-Verbose "Searching $Path for *.cobertura.xml files" +$reports = Get-ChildItem -Recurse $Path -Filter *.cobertura.xml + +if ($reports) { + $reports |% { $_.FullName } |% { + # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. + $xml = [xml](Get-Content -Path $_) + $xml.coverage.packages.package.classes.class |? { $_.filename} |% { + $_.filename = $_.filename.Replace('{reporoot}', $RepoRoot).Replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) + } + + $xml.Save($_) + } + + $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_.FullName })) + & "$RepoRoot/obj/reportgenerator" -reports:"$Inputs" -targetdir:$OutputDir -reporttypes:$Format +} else { + Write-Error "No reports found to merge." +} diff --git a/azure-pipelines/PoliCheckExclusions.xml b/azure-pipelines/PoliCheckExclusions.xml new file mode 100644 index 00000000..5ae16710 --- /dev/null +++ b/azure-pipelines/PoliCheckExclusions.xml @@ -0,0 +1,10 @@ + + + NODE_MODULES|.STORE + + + + + + + diff --git a/azure-pipelines/Publish-Legacy-Symbols.ps1 b/azure-pipelines/Publish-Legacy-Symbols.ps1 new file mode 100644 index 00000000..5c4035a1 --- /dev/null +++ b/azure-pipelines/Publish-Legacy-Symbols.ps1 @@ -0,0 +1,37 @@ +Param( + [string]$Path +) + +$ArtifactStagingFolder = & "$PSScriptRoot/Get-ArtifactsStagingDirectory.ps1" +$ArtifactStagingFolder += '/symbols-legacy' +robocopy $Path $ArtifactStagingFolder /mir /njh /njs /ndl /nfl +$WindowsPdbSubDirName = 'symstore' + +Get-ChildItem "$ArtifactStagingFolder\*.pdb" -Recurse |% { + $dllPath = "$($_.Directory)/$($_.BaseName).dll" + $exePath = "$($_.Directory)/$($_.BaseName).exe" + if (Test-Path $dllPath) { + $BinaryImagePath = $dllPath + } elseif (Test-Path $exePath) { + $BinaryImagePath = $exePath + } else { + Write-Warning "`"$_`" found with no matching binary file." + $BinaryImagePath = $null + } + + if ($BinaryImagePath) { + # Convert the PDB to legacy Windows PDBs + Write-Host "Converting PDB for $_" -ForegroundColor DarkGray + $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" + if (!(Test-Path $WindowsPdbDir)) { mkdir $WindowsPdbDir | Out-Null } + $legacyPdbPath = "$WindowsPdbDir\$($_.BaseName).pdb" + & "$PSScriptRoot\Convert-PDB.ps1" -DllPath $BinaryImagePath -PdbPath $_ -OutputPath $legacyPdbPath + if ($LASTEXITCODE -ne 0) { + Write-Warning "PDB conversion of `"$_`" failed." + } + + Move-Item $legacyPdbPath $_ -Force + } +} + +Write-Host "##vso[artifact.upload containerfolder=symbols-legacy;artifactname=symbols-legacy;]$ArtifactStagingFolder" diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index 9c9c4250..ba6af320 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -1,5 +1,15 @@ # This artifact captures everything needed to insert into VS (NuGet packages, insertion metadata, etc.) +[CmdletBinding()] +Param ( +) + +if ($IsMacOS -or $IsLinux) { + # We only package up for insertions on Windows agents since they are where optprof can happen. + Write-Verbose "Skipping VSInsertion artifact since we're not on Windows." + return @{} +} + $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") $BuildConfiguration = $env:BUILDCONFIGURATION if (!$BuildConfiguration) { @@ -8,7 +18,10 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" -if (!(Test-Path $PackagesRoot)) { return } +if (!(Test-Path $PackagesRoot)) { + Write-Warning "Skipping because packages haven't been built yet." + return @{} +} @{ "$PackagesRoot" = (Get-ChildItem $PackagesRoot -Recurse) diff --git a/azure-pipelines/artifacts/_all.ps1 b/azure-pipelines/artifacts/_all.ps1 index afe42be3..033cc87c 100755 --- a/azure-pipelines/artifacts/_all.ps1 +++ b/azure-pipelines/artifacts/_all.ps1 @@ -12,9 +12,10 @@ Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact. FileInfo objects are also allowed. .PARAMETER Force - Executes artifact scripts even if they have already been uploaded. + Executes artifact scripts even if they have already been staged. #> +[CmdletBinding(SupportsShouldProcess = $true)] param ( [string]$ArtifactNameSuffix, [switch]$Force @@ -28,14 +29,14 @@ Function EnsureTrailingSlash($path) { $path.Replace('\', [IO.Path]::DirectorySeparatorChar) } -Function Test-ArtifactUploaded($artifactName) { - $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())" +Function Test-ArtifactStaged($artifactName) { + $varName = "ARTIFACTSTAGED_$($artifactName.ToUpper())" Test-Path "env:$varName" } Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % { $ArtifactName = $_.BaseName - if ($Force -or !(Test-ArtifactUploaded($ArtifactName + $ArtifactNameSuffix))) { + if ($Force -or !(Test-ArtifactStaged($ArtifactName + $ArtifactNameSuffix))) { $totalFileCount = 0 $fileGroups = & $_ if ($fileGroups) { @@ -65,6 +66,6 @@ Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse | % { Write-Warning "No files found for the `"$ArtifactName`" artifact." } } else { - Write-Host "Skipping $ArtifactName because it has already been uploaded." -ForegroundColor DarkGray + Write-Host "Skipping $ArtifactName because it has already been staged." -ForegroundColor DarkGray } } diff --git a/azure-pipelines/artifacts/_pipelines.ps1 b/azure-pipelines/artifacts/_pipelines.ps1 index 73a3af0a..2d3338b2 100644 --- a/azure-pipelines/artifacts/_pipelines.ps1 +++ b/azure-pipelines/artifacts/_pipelines.ps1 @@ -1,15 +1,44 @@ -# This script translates all the artifacts described by _all.ps1 -# into commands that instruct Azure Pipelines to actually collect those artifacts. +<# +.SYNOPSIS + This script translates all the artifacts described by _all.ps1 + into commands that instruct Azure Pipelines to actually collect those artifacts. +#> +[CmdletBinding()] param ( - [string]$ArtifactNameSuffix + [string]$ArtifactNameSuffix, + [switch]$StageOnly ) -& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { - Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)" +Function Set-PipelineVariable($name, $value) { + if ((Test-Path "Env:\$name") -and (Get-Item "Env:\$name").Value -eq $value) { + return # already set + } + + #New-Item -Path "Env:\$name".ToUpper() -Value $value -Force | Out-Null + Write-Host "##vso[task.setvariable variable=$name]$value" +} + +Function Test-ArtifactUploaded($artifactName) { + $varName = "ARTIFACTUPLOADED_$($artifactName.ToUpper())" + Test-Path "env:$varName" +} +& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts # will skip this one from a check in the _all.ps1 script. - $varName = "ARTIFACTUPLOADED_$($_.Name.ToUpper())" - Write-Host "##vso[task.setvariable variable=$varName]true" + Set-PipelineVariable "ARTIFACTSTAGED_$($_.Name.ToUpper())" 'true' + Write-Host "Staged artifact $($_.Name) to $($_.Path)" + + if (!$StageOnly) { + if (Test-ArtifactUploaded $_.Name) { + Write-Host "Skipping $($_.Name) because it has already been uploaded." -ForegroundColor DarkGray + } else { + Write-Host "##vso[artifact.upload containerfolder=$($_.Name);artifactname=$($_.Name);]$($_.Path)" + + # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts + # will skip this one from a check in the _all.ps1 script. + Set-PipelineVariable "ARTIFACTUPLOADED_$($_.Name.ToUpper())" 'true' + } + } } diff --git a/azure-pipelines/artifacts/_stage_all.ps1 b/azure-pipelines/artifacts/_stage_all.ps1 index e4954c13..d81d16d4 100644 --- a/azure-pipelines/artifacts/_stage_all.ps1 +++ b/azure-pipelines/artifacts/_stage_all.ps1 @@ -1,6 +1,10 @@ -# This script links all the artifacts described by _all.ps1 -# into a staging directory, reading for uploading to a cloud build artifact store. -# It returns a sequence of objects with Name and Path properties. +<# +.SYNOPSIS + This script links all the artifacts described by _all.ps1 + into a staging directory, reading for uploading to a cloud build artifact store. + It returns a sequence of objects with Name and Path properties. +#> + [CmdletBinding()] param ( [string]$ArtifactNameSuffix @@ -42,7 +46,13 @@ $Artifacts |% { } } -$Artifacts |% { "$($_.ArtifactName)$ArtifactNameSuffix" } | Get-Unique |% { +$ArtifactNames = $Artifacts |% { "$($_.ArtifactName)$ArtifactNameSuffix" } +$ArtifactNames += Get-ChildItem env:ARTIFACTSTAGED_* |% { + # Return from ALLCAPS to the actual capitalization used for the artifact. + $artifactNameAllCaps = "$($_.Name.Substring('ARTIFACTSTAGED_'.Length))" + (Get-ChildItem $ArtifactStagingFolder\$artifactNameAllCaps* -Filter $artifactNameAllCaps).Name +} +$ArtifactNames | Get-Unique |% { $artifact = New-Object -TypeName PSObject Add-Member -InputObject $artifact -MemberType NoteProperty -Name Name -Value $_ Add-Member -InputObject $artifact -MemberType NoteProperty -Name Path -Value (Join-Path $ArtifactStagingFolder $_) diff --git a/azure-pipelines/artifacts/coverageResults.ps1 b/azure-pipelines/artifacts/coverageResults.ps1 index 8fdb3f72..280ff9ae 100644 --- a/azure-pipelines/artifacts/coverageResults.ps1 +++ b/azure-pipelines/artifacts/coverageResults.ps1 @@ -1,10 +1,11 @@ $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") +$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse | Where {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) + # Prepare code coverage reports for merging on another machine if ($env:SYSTEM_DEFAULTWORKINGDIRECTORY) { Write-Host "Substituting $env:SYSTEM_DEFAULTWORKINGDIRECTORY with `"{reporoot}`"" - $reports = Get-ChildItem "$RepoRoot/bin/coverage.*cobertura.xml" -Recurse - $reports |% { + $coverageFiles |% { $content = Get-Content -Path $_ |% { $_ -Replace [regex]::Escape($env:SYSTEM_DEFAULTWORKINGDIRECTORY), "{reporoot}" } Set-Content -Path $_ -Value $content -Encoding UTF8 } @@ -16,7 +17,7 @@ if (!((Test-Path $RepoRoot\bin) -and (Test-Path $RepoRoot\obj))) { return } @{ $RepoRoot = ( - @(Get-ChildItem "$RepoRoot\bin\coverage.*cobertura.xml" -Recurse) + + $coverageFiles + (Get-ChildItem "$RepoRoot\obj\*.cs" -Recurse) ); } diff --git a/azure-pipelines/artifacts/testResults.ps1 b/azure-pipelines/artifacts/testResults.ps1 index 2f894c97..301a4376 100644 --- a/azure-pipelines/artifacts/testResults.ps1 +++ b/azure-pipelines/artifacts/testResults.ps1 @@ -1,16 +1,11 @@ +[CmdletBinding()] +Param( +) + $result = @{} -if ($env:AGENT_TEMPDIRECTORY) { - # The DotNetCoreCLI uses an alternate location to publish these files - $guidRegex = '^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$' - $result[$env:AGENT_TEMPDIRECTORY] = (Get-ChildItem $env:AGENT_TEMPDIRECTORY -Directory |? { $_.Name -match $guidRegex } |% { - Get-ChildItem "$($_.FullName)\dotnet*.dmp","$($_.FullName)\*_crashdump.dmp","$($_.FullName)\testhost*.dmp","$($_.FullName)\Sequence_*.xml" -Recurse - }); -} -else { - $testRoot = Resolve-Path "$PSScriptRoot\..\..\test" - $result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) -} +$testRoot = Resolve-Path "$PSScriptRoot\..\..\test" +$result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) $testlogsPath = "$env:BUILD_ARTIFACTSTAGINGDIRECTORY\test_logs" if (Test-Path $testlogsPath) { diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 0acc0069..e6df1321 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -4,6 +4,9 @@ parameters: default: vmImage: windows-2022 - name: includeMacOS +- name: RunTests + type: boolean + default: true jobs: - job: Windows @@ -18,10 +21,12 @@ jobs: - template: install-dependencies.yml - - powershell: '& (./azure-pipelines/Get-nbgv.ps1) cloud -c' + - powershell: '& (./azure-pipelines/Get-nbgv.ps1) cloud -ca' displayName: โš™ Set build number - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - job: Linux timeoutInMinutes: 120 @@ -36,6 +41,8 @@ jobs: submodules: true - template: install-dependencies.yml - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - job: macOS condition: ${{ parameters.includeMacOS }} @@ -51,6 +58,8 @@ jobs: submodules: true - template: install-dependencies.yml - template: dotnet.yml + parameters: + RunTests: ${{ parameters.RunTests }} - job: WrapUp dependsOn: @@ -68,7 +77,7 @@ jobs: - template: publish-symbols.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} - - template: publish-codecoverage.yml - parameters: - includeMacOS: ${{ parameters.includeMacOS }} - - template: publish-deployables.yml + - ${{ if parameters.RunTests }}: + - template: publish-codecoverage.yml + parameters: + includeMacOS: ${{ parameters.includeMacOS }} diff --git a/azure-pipelines/dotnet-test-cloud.ps1 b/azure-pipelines/dotnet-test-cloud.ps1 index f636636f..d1a06080 100755 --- a/azure-pipelines/dotnet-test-cloud.ps1 +++ b/azure-pipelines/dotnet-test-cloud.ps1 @@ -1,24 +1,60 @@ #!/usr/bin/env pwsh +<# +.SYNOPSIS + Runs tests as they are run in cloud test runs. +.PARAMETER Configuration + The configuration within which to run tests +.PARAMETER Agent + The name of the agent. This is used in preparing test run titles. +.PARAMETER PublishResults + A switch to publish results to Azure Pipelines. +.PARAMETER x86 + A switch to run the tests in an x86 process. +.PARAMETER dotnet32 + The path to a 32-bit dotnet executable to use. +#> +[CmdletBinding()] Param( [string]$Configuration='Debug', [string]$Agent='Local', - [switch]$PublishResults + [switch]$PublishResults, + [switch]$x86, + [string]$dotnet32 ) $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path $ArtifactStagingFolder = & "$PSScriptRoot/Get-ArtifactsStagingDirectory.ps1" -dotnet test $RepoRoot ` +$dotnet = 'dotnet' +if ($x86) { + $x86RunTitleSuffix = ", x86" + if ($dotnet32) { + $dotnet = $dotnet32 + } else { + $dotnet32Possibilities = "$PSScriptRoot\../obj/tools/x86/.dotnet/dotnet.exe", "$env:AGENT_TOOLSDIRECTORY/x86/dotnet/dotnet.exe", "${env:ProgramFiles(x86)}\dotnet\dotnet.exe" + $dotnet32Matches = $dotnet32Possibilities |? { Test-Path $_ } + if ($dotnet32Matches) { + $dotnet = Resolve-Path @($dotnet32Matches)[0] + Write-Host "Running tests using `"$dotnet`"" -ForegroundColor DarkGray + } else { + Write-Error "Unable to find 32-bit dotnet.exe" + return 1 + } + } +} + +& $dotnet test $RepoRoot ` --no-build ` -c $Configuration ` --filter "TestCategory!=FailsInCloudTest$env:TESTFILTER" ` - -p:CollectCoverage=true ` + --collect "Code Coverage;Format=cobertura" ` + --settings "$PSScriptRoot/test.runsettings" ` --blame-hang-timeout 600s ` --blame-crash ` -bl:"$ArtifactStagingFolder/build_logs/test.binlog" ` --diag "$ArtifactStagingFolder/test_logs/diag.log;TraceLevel=info" ` - --logger trx + --logger trx ` $unknownCounter = 0 Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { @@ -33,13 +69,13 @@ Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { if ($matches.rid) { $runTitle = "$($matches.lib) ($($matches.tfm), $($matches.rid), $Agent)" } else { - $runTitle = "$($matches.lib) ($($matches.tfm), $Agent)" + $runTitle = "$($matches.lib) ($($matches.tfm)$x86RunTitleSuffix, $Agent)" } } } if (!$runTitle) { $unknownCounter += 1; - $runTitle = "unknown$unknownCounter ($Agent)"; + $runTitle = "unknown$unknownCounter ($Agent$x86RunTitleSuffix)"; } Write-Host "##vso[results.publish type=VSTest;runTitle=$runTitle;publishRunAttachments=true;resultFiles=$_;failTaskOnFailedTests=true;testRunSystem=VSTS - PTR;]" diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 6fa04354..88c68c00 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,3 +1,6 @@ +parameters: + RunTests: + steps: - script: dotnet build /t:build --no-restore -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog" displayName: ๐Ÿ›  dotnet build @@ -13,18 +16,22 @@ steps: - powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults displayName: ๐Ÿงช dotnet test + condition: and(succeeded(), ${{ parameters.RunTests }}) - powershell: azure-pipelines/variables/_pipelines.ps1 failOnStderr: true displayName: โš™ Update pipeline variables based on build outputs condition: succeededOrFailed() -- powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" +- powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose failOnStderr: true displayName: ๐Ÿ“ข Publish artifacts condition: succeededOrFailed() -- bash: bash <(curl -s https://codecov.io/bash) +- powershell: | + $ArtifactStagingFolder = & "azure-pipelines/Get-ArtifactsStagingDirectory.ps1" + $CoverageResultsFolder = Join-Path $ArtifactStagingFolder "coverageResults-$(Agent.JobName)" + azure-pipelines/publish-CodeCov.ps1 -CodeCovToken "$(codecov_token)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)Host,$(BuildConfiguration)" displayName: ๐Ÿ“ข Publish code coverage results to codecov.io condition: ne(variables['codecov_token'], '') timeoutInMinutes: 3 diff --git a/azure-pipelines/falsepositives.gdnsuppress b/azure-pipelines/falsepositives.gdnsuppress new file mode 100644 index 00000000..1248172b --- /dev/null +++ b/azure-pipelines/falsepositives.gdnsuppress @@ -0,0 +1,12 @@ +{ + "version": "latest", + "suppressionSets": { + "falsepositives": { + "name": "falsepositives", + "createdDate": "2021-12-03 00:23:08Z", + "lastUpdatedDate": "2021-12-03 00:23:08Z" + } + }, + "results": { + } +} diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 1843cb97..59dfcf20 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -25,9 +25,10 @@ parameters: displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines - -variables: - NugetSecurityAnalysisWarningLevel: none # nuget.config requires signed packages by trusted owners +- name: RunTests + displayName: Run tests + type: boolean + default: true stages: @@ -37,40 +38,35 @@ stages: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release BuildPlatform: Any CPU - push_to_ci: true - ci_feed: https://pkgs.dev.azure.com/microsoft/_packaging/MSFTNuget/nuget/v3/index.json - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ SignSelection: ${{ parameters.SignSelection }} jobs: - template: build.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} + RunTests: ${{ parameters.RunTests }} - stage: azure_public_winsdk_feed displayName: azure-public/winsdk feed condition: and(succeeded(), eq(dependencies.Build.outputs['Windows.SetPipelineVariables.IsSigned'], 'true')) jobs: - - deployment: push + - job: push pool: vmImage: ubuntu-latest - environment: No-Approval - strategy: - runOnce: - deploy: - steps: - - download: current - artifact: deployables-Windows - displayName: Download deployables-Windows artifact - - task: NuGetToolInstaller@1 - displayName: Use NuGet 5.x - inputs: - versionSpec: 5.x - - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 # NuGetCommand - displayName: NuGet push - inputs: - command: push - packagesToPush: $(Pipeline.Workspace)/deployables-Windows/*.nupkg - nuGetFeedType: external - publishFeedCredentials: azure-public/winsdk - continueOnError: true # until "skip on conflict" is offered as a task input. + steps: + - checkout: none + - download: current + artifact: deployables-Windows + displayName: Download deployables-Windows artifact + - task: UseDotNet@2 + displayName: Install .NET SDK + inputs: + packageType: sdk + version: 6.x + - task: NuGetAuthenticate@0 + displayName: Authenticate NuGet feeds + inputs: + nuGetServiceConnections: azure-public/winsdk + forceReinstallCredentialProvider: true + - script: dotnet nuget push $(Pipeline.Workspace)/deployables-Windows/*.nupkg -s https://pkgs.dev.azure.com/azure-public/winsdk/_packaging/CI/nuget/v3/index.json --api-key azdo --skip-duplicate diff --git a/azure-pipelines/publish-CodeCov.ps1 b/azure-pipelines/publish-CodeCov.ps1 new file mode 100644 index 00000000..9926f018 --- /dev/null +++ b/azure-pipelines/publish-CodeCov.ps1 @@ -0,0 +1,30 @@ +<# +.SYNOPSIS + Uploads code coverage to codecov.io +.PARAMETER CodeCovToken + Code coverage token to use +.PARAMETER PathToCodeCoverage + Path to root of code coverage files +.PARAMETER Name + Name to upload with codecoverge +.PARAMETER Flags + Flags to upload with codecoverge +#> +[CmdletBinding()] +Param ( + [Parameter(Mandatory=$true)] + [string]$CodeCovToken, + [Parameter(Mandatory=$true)] + [string]$PathToCodeCoverage, + [string]$Name, + [string]$Flags +) + +$RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path + +Get-ChildItem -Recurse -Path $PathToCodeCoverage -Filter "*.cobertura.xml" | % { + $relativeFilePath = Resolve-Path -relative $_.FullName + + Write-Host "Uploading: $relativeFilePath" -ForegroundColor Yellow + & (& "$PSScriptRoot/Get-CodeCovTool.ps1") -t $CodeCovToken -f $relativeFilePath -R $RepoRoot -F $Flags -n $Name +} diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index 423e12fc..11de8ffa 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -14,22 +14,12 @@ steps: artifact: coverageResults-macOS displayName: ๐Ÿ”ป Download macOS code coverage results continueOnError: true - condition: ${{ parameters.includeMacOS }} -- powershell: | - dotnet tool install --tool-path obj dotnet-reportgenerator-globaltool --version 4.8.5 --configfile azure-pipelines/justnugetorg.nuget.config - Copy-Item -Recurse $(Pipeline.Workspace)/coverageResults-Windows/obj/* $(System.DefaultWorkingDirectory)/obj - Write-Host 'Substituting {reporoot} with $(System.DefaultWorkingDirectory)' - $reports = Get-ChildItem -Recurse '$(Pipeline.Workspace)/coverage.*cobertura.xml' - $reports |% { - $content = Get-Content -Path $_ |% { $_.Replace('{reporoot}', '$(System.DefaultWorkingDirectory)') } - Set-Content -Path $_ -Value $content -Encoding UTF8 - } - $Inputs = [string]::join(';', ($reports |% { Resolve-Path -relative $_ })) - obj/reportgenerator -reports:"$Inputs" -targetdir:coveragereport -reporttypes:Cobertura + condition: and(succeeded(), ${{ parameters.includeMacOS }}) +- powershell: azure-pipelines/Merge-CodeCoverage.ps1 -Path '$(Pipeline.Workspace)' -OutputDir coveragereport -Format Cobertura -Verbose displayName: โš™ Merge coverage - task: PublishCodeCoverageResults@1 displayName: ๐Ÿ“ข Publish code coverage results to Azure DevOps inputs: codeCoverageTool: cobertura - summaryFileLocation: 'coveragereport/Cobertura.xml' + summaryFileLocation: coveragereport/Cobertura.xml failIfCoverageEmpty: true diff --git a/azure-pipelines/publish-deployables.yml b/azure-pipelines/publish-deployables.yml deleted file mode 100644 index 31e80a43..00000000 --- a/azure-pipelines/publish-deployables.yml +++ /dev/null @@ -1,8 +0,0 @@ -steps: -- download: current - displayName: ๐Ÿ”ป Download deployables - artifact: deployables-Windows - -- powershell: dotnet nuget push "$(Resolve-Path '$(Pipeline.Workspace)\deployables-Windows\')*.nupkg" -s $(ci_feed) -k azdo --skip-duplicate - displayName: ๐Ÿ“ฆ Push packages to CI feed - condition: and(succeeded(), ne(variables['ci_feed'], ''), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/azure-pipelines/publish-symbols.yml b/azure-pipelines/publish-symbols.yml index 00c188fc..00b5a3a6 100644 --- a/azure-pipelines/publish-symbols.yml +++ b/azure-pipelines/publish-symbols.yml @@ -57,3 +57,6 @@ steps: IndexSources: false SymbolServerType: TeamServices displayName: ๐Ÿ“ข Publish test symbols + +- powershell: azure-pipelines/Publish-Legacy-Symbols.ps1 -Path $(Pipeline.Workspace)/symbols/Windows + displayName: ๐Ÿ“ข Publish symbols for symbol archival diff --git a/azure-pipelines/release-deployment-prep.yml b/azure-pipelines/release-deployment-prep.yml index 6dee28e5..059fd1d0 100644 --- a/azure-pipelines/release-deployment-prep.yml +++ b/azure-pipelines/release-deployment-prep.yml @@ -3,7 +3,7 @@ steps: artifact: Variables-Windows displayName: Download Variables-Windows artifact - task: PowerShell@2 - displayName: Set VSTS variables based on artifacts + displayName: Set pipeline variables based on artifacts inputs: targetType: filePath filePath: $(Pipeline.Workspace)/CI/Variables-Windows/_pipelines.ps1 diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index fa9a00d2..25d36444 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -46,14 +46,15 @@ jobs: title: v$(resources.pipeline.CI.runName) isDraft: true # After running this step, visit the new draft release, edit, and publish. isPreRelease: $(IsPrerelease) - assets: $(Pipeline.Workspace)/CI/deployables-Windows/*.nupkg + assets: $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg changeLogCompareToRelease: lastNonDraftRelease changeLogType: issueBased changeLogLabels: | [ + { "label" : "breaking change", "displayName" : "Breaking changes", "state" : "closed" }, { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } ] - - script: dotnet nuget push $(Pipeline.Workspace)/CI/deployables-Windows/*.nupkg -s https://api.nuget.org/v3/index.json --api-key $(NuGetOrgApiKey) --skip-duplicate + - script: dotnet nuget push $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg -s https://api.nuget.org/v3/index.json --api-key $(NuGetOrgApiKey) --skip-duplicate displayName: ๐Ÿ“ฆ Push packages to nuget.org condition: and(succeeded(), ne(variables['NuGetOrgApiKey'], '')) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml deleted file mode 100644 index 68b5d5f8..00000000 --- a/azure-pipelines/secure-development-tools.yml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - - ### Check for checked in credentials. -- task: CredScan@3 - displayName: 'Run CredScan' - condition: false # VERY slow due to our checked out submodules - - ### Run PoliCheck to check for disallowed terms. targetType: F indicates we're searching files and folders. -- task: PoliCheck@1 - displayName: 'Run PoliCheck' - inputs: - targetType: F - condition: false # VERY slow due to our checked out submodules diff --git a/azure-pipelines/test.runsettings b/azure-pipelines/test.runsettings new file mode 100644 index 00000000..c69022fc --- /dev/null +++ b/azure-pipelines/test.runsettings @@ -0,0 +1,44 @@ + + + + + + + + + \.dll$ + \.exe$ + + + xunit\..* + + + + + ^System\.Diagnostics\.DebuggerHiddenAttribute$ + ^System\.Diagnostics\.DebuggerNonUserCodeAttribute$ + ^System\.CodeDom\.Compiler\.GeneratedCodeAttribute$ + ^System\.Diagnostics\.CodeAnalysis\.ExcludeFromCodeCoverageAttribute$ + + + + + True + + True + + True + + False + + True + + True + + True + + + + + + diff --git a/azure-pipelines/variables/InsertConfigValues.ps1 b/azure-pipelines/variables/InsertConfigValues.ps1 index 1455a5b7..3ae11de9 100644 --- a/azure-pipelines/variables/InsertConfigValues.ps1 +++ b/azure-pipelines/variables/InsertConfigValues.ps1 @@ -1,9 +1,9 @@ -$BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin\Packages\$env:BUILDCONFIGURATION") +$InsertedPkgs = (& "$PSScriptRoot\..\artifacts\VSInsertion.ps1") $icv=@() -if (Test-Path "$BinPath") { - Get-ChildItem -Path "$BinPath\*.nupkg" -rec |% { - if ($_.Name -match "^(.*)\.(\d+\.\d+\.\d+(?:-.*?)?)(?:\.symbols)?\.nupkg$") { +foreach ($kvp in $InsertedPkgs.GetEnumerator()) { + $kvp.Value |% { + if ($_.Name -match "^(.*?)\.(\d+\.\d+\.\d+(?:\.\d+)?(?:-.*?)?)(?:\.symbols)?\.nupkg$") { $id = $Matches[1] $version = $Matches[2] $icv += "$id=$version" diff --git a/azure-pipelines/variables/TeamName.ps1 b/azure-pipelines/variables/TeamName.ps1 index 871749a5..5f2822c5 100644 --- a/azure-pipelines/variables/TeamName.ps1 +++ b/azure-pipelines/variables/TeamName.ps1 @@ -1 +1,2 @@ +# This value is used to craft a \\cpvsbuild\drops path for symbol archival. 'VS IDE' diff --git a/azure-pipelines/variables/_all.ps1 b/azure-pipelines/variables/_all.ps1 index 0407d307..cc6e8810 100755 --- a/azure-pipelines/variables/_all.ps1 +++ b/azure-pipelines/variables/_all.ps1 @@ -1,7 +1,14 @@ #!/usr/bin/env pwsh -# This script returns a hashtable of build variables that should be set -# at the start of a build or release definition's execution. +<# +.SYNOPSIS + This script returns a hashtable of build variables that should be set + at the start of a build or release definition's execution. +#> + +[CmdletBinding(SupportsShouldProcess = $true)] +param ( +) $vars = @{} diff --git a/azure-pipelines/variables/_pipelines.ps1 b/azure-pipelines/variables/_pipelines.ps1 index 867b7fc8..951106d2 100644 --- a/azure-pipelines/variables/_pipelines.ps1 +++ b/azure-pipelines/variables/_pipelines.ps1 @@ -1,8 +1,15 @@ -# This script translates the variables returned by the _all.ps1 script -# into commands that instruct Azure Pipelines to actually set those variables for other pipeline tasks to consume. +<# +.SYNOPSIS + This script translates the variables returned by the _all.ps1 script + into commands that instruct Azure Pipelines to actually set those variables for other pipeline tasks to consume. -# The build or release definition may have set these variables to override -# what the build would do. So only set them if they have not already been set. + The build or release definition may have set these variables to override + what the build would do. So only set them if they have not already been set. +#> + +[CmdletBinding()] +param ( +) (& "$PSScriptRoot\_all.ps1").GetEnumerator() |% { # Always use ALL CAPS for env var names since Azure Pipelines converts variable names to all caps and on non-Windows OS, env vars are case sensitive. diff --git a/nuget.config b/nuget.config index f4e66465..0a73357e 100644 --- a/nuget.config +++ b/nuget.config @@ -1,26 +1,12 @@ - +๏ปฟ - - - - + - - - Microsoft;xunit;manuel.roemer;sharwell;jamesnk;aarnott;MarcoRossignoli;Thecentury;aaubry;mmanela;onovotny - - - - - - - - diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index 2faab375..e7edee55 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -1,10 +1,3 @@ - - cobertura - [xunit.*]* - - $(OutputPath)/ - - diff --git a/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj b/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj index e1b6a7ee..f0766fc2 100644 --- a/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj +++ b/test/GenerationSandbox.Tests/GenerationSandbox.Tests.csproj @@ -30,9 +30,9 @@ - + - + diff --git a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs index 6ee820cc..507e86ea 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs +++ b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs @@ -206,20 +206,20 @@ public void TemplateProvidedMembersMatchVisibilityWithContainingType_OtherMember this.CollectGeneratedCode(this.generator); this.AssertNoDiagnostics(); - StructDeclarationSyntax? pcstrType = (StructDeclarationSyntax?)this.FindGeneratedType("PCSTR").Single(); + StructDeclarationSyntax pcstrType = (StructDeclarationSyntax)this.FindGeneratedType("PCSTR").Single(); SyntaxKind expectedVisibility = generatePublic ? SyntaxKind.PublicKeyword : SyntaxKind.InternalKeyword; // Assert fields - Assert.Contains(pcstrType?.Members.OfType(), f => f.Declaration.Variables.Any(v => v.Identifier.ValueText == "Value") && f.Modifiers.Any(expectedVisibility)); + Assert.Contains(pcstrType.Members.OfType(), f => f.Declaration.Variables.Any(v => v.Identifier.ValueText == "Value") && f.Modifiers.Any(expectedVisibility)); // Assert properties - Assert.Contains(pcstrType?.Members.OfType(), p => p.Identifier.ValueText == "Length" && p.Modifiers.Any(expectedVisibility)); + Assert.Contains(pcstrType.Members.OfType(), p => p.Identifier.ValueText == "Length" && p.Modifiers.Any(expectedVisibility)); // Assert constructors - Assert.All(pcstrType?.Members.OfType(), c => c.Modifiers.Any(expectedVisibility)); + Assert.All(pcstrType.Members.OfType(), c => c.Modifiers.Any(expectedVisibility)); // Assert that private members remain private. - Assert.Contains(pcstrType?.Members.OfType(), p => p.Identifier.ValueText == "DebuggerDisplay" && p.Modifiers.Any(SyntaxKind.PrivateKeyword)); + Assert.Contains(pcstrType.Members.OfType(), p => p.Identifier.ValueText == "DebuggerDisplay" && p.Modifiers.Any(SyntaxKind.PrivateKeyword)); } [Fact] @@ -523,8 +523,8 @@ public void WinRTInterfaceWithWinRTOutObjectUsesMarshaler() AttributeSyntax marshalAsAttr = Assert.Single(FindAttribute(lastParam.AttributeLists, "MarshalAs")); Assert.True(marshalAsAttr.ArgumentList?.Arguments[0].ToString() == "UnmanagedType.CustomMarshaler"); - Assert.Single(marshalAsAttr.ArgumentList?.Arguments.Where(arg => arg.ToString() == $"MarshalCookie = \"{WinRTClassName}\"")); - Assert.Single(marshalAsAttr.ArgumentList?.Arguments.Where(arg => arg.ToString() == $"MarshalType = \"{WinRTCustomMarshalerFullName}\"")); + Assert.Single(marshalAsAttr.ArgumentList.Arguments.Where(arg => arg.ToString() == $"MarshalCookie = \"{WinRTClassName}\"")); + Assert.Single(marshalAsAttr.ArgumentList.Arguments.Where(arg => arg.ToString() == $"MarshalType = \"{WinRTCustomMarshalerFullName}\"")); // Make sure the WinRT marshaler was brought in Assert.Single(this.FindGeneratedType(WinRTCustomMarshalerClass)); diff --git a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj index 9cdfee39..6a0bc18b 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj +++ b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj @@ -42,11 +42,11 @@ - + - + diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 index 7dcddc78..2bac3b9b 100644 --- a/tools/Install-DotNetSdk.ps1 +++ b/tools/Install-DotNetSdk.ps1 @@ -15,20 +15,28 @@ When using 'repo', environment variables are set to cause the locally installed dotnet SDK to be used. Per-repo can lead to file locking issues when dotnet.exe is left running as a build server and can be mitigated by running `dotnet build-server shutdown`. Per-machine requires elevation and will download and install all SDKs and runtimes to machine-wide locations so all applications can find it. +.PARAMETER IncludeX86 + Installs a x86 SDK and runtimes in addition to the x64 ones. Only supported on Windows. Ignored on others. #> [CmdletBinding(SupportsShouldProcess=$true,ConfirmImpact='Medium')] Param ( [ValidateSet('repo','user','machine')] - [string]$InstallLocality='user' + [string]$InstallLocality='user', + [switch]$IncludeX86 ) $DotNetInstallScriptRoot = "$PSScriptRoot/../obj/tools" if (!(Test-Path $DotNetInstallScriptRoot)) { New-Item -ItemType Directory -Path $DotNetInstallScriptRoot -WhatIf:$false | Out-Null } $DotNetInstallScriptRoot = Resolve-Path $DotNetInstallScriptRoot -# Look up actual required .NET Core SDK version from global.json +# Look up actual required .NET SDK version from global.json $sdkVersion = & "$PSScriptRoot/../azure-pipelines/variables/DotNetSdkVersion.ps1" +If ($IncludeX86 -and ($IsMacOS -or $IsLinux)) { + Write-Verbose "Ignoring -IncludeX86 switch because 32-bit runtimes are only supported on Windows." + $IncludeX86 = $false +} + $arch = [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture if (!$arch) { # Windows Powershell leaves this blank $arch = 'x64' @@ -72,7 +80,7 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile = Join-Path $OutDir $Uri.Segments[-1] if (!(Test-Path $OutFile)) { Write-Verbose "Downloading $Uri..." - if (!(Test-Path $OutDir)) { mkdir $OutDir } + if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir | Out-Null } try { (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) } finally { @@ -83,40 +91,36 @@ Function Get-FileFromWeb([Uri]$Uri, $OutDir) { $OutFile } -Function Get-InstallerExe($Version, [switch]$Runtime) { - $sdkOrRuntime = 'Sdk' - if ($Runtime) { $sdkOrRuntime = 'Runtime' } - +Function Get-InstallerExe( + $Version, + $Architecture, + [ValidateSet('Sdk','Runtime','WindowsDesktop')] + [string]$sku +) { # Get the latest/actual version for the specified one $TypedVersion = [Version]$Version if ($TypedVersion.Build -eq -1) { - $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sdkOrRuntime/$Version/latest.version" -UseBasicParsing) + $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sku/$Version/latest.version" -UseBasicParsing) $Version = $versionInfo[-1] } $majorMinor = "$($TypedVersion.Major).$($TypedVersion.Minor)" $ReleasesFile = Join-Path $DotNetInstallScriptRoot "$majorMinor\releases.json" if (!(Test-Path $ReleasesFile)) { - Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) + Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null } $releases = Get-Content $ReleasesFile | ConvertFrom-Json $url = $null foreach ($release in $releases.releases) { $filesElement = $null - if ($Runtime) { - if ($release.runtime.version -eq $Version) { - $filesElement = $release.runtime.files - } - } else { - if ($release.sdk.version -eq $Version) { - $filesElement = $release.sdk.files - } + if ($release.$sku.version -eq $Version) { + $filesElement = $release.$sku.files } if ($filesElement) { foreach ($file in $filesElement) { - if ($file.rid -eq "win-$arch") { + if ($file.rid -eq "win-$Architecture") { $url = $file.url Break } @@ -131,26 +135,23 @@ Function Get-InstallerExe($Version, [switch]$Runtime) { if ($url) { Get-FileFromWeb -Uri $url -OutDir $DotNetInstallScriptRoot } else { - Write-Error "Unable to find release of $sdkOrRuntime v$Version" + Write-Error "Unable to find release of $sku v$Version" } } -Function Install-DotNet($Version, [switch]$Runtime) { - if ($Runtime) { $sdkSubstring = '' } else { $sdkSubstring = 'SDK ' } - Write-Host "Downloading .NET Core $sdkSubstring$Version..." - $Installer = Get-InstallerExe -Version $Version -Runtime:$Runtime - Write-Host "Installing .NET Core $sdkSubstring$Version..." +Function Install-DotNet($Version, $Architecture, [ValidateSet('Sdk','Runtime','WindowsDesktop')][string]$sku = 'Sdk') { + Write-Host "Downloading .NET Core $sku $Version..." + $Installer = Get-InstallerExe -Version $Version -Architecture $Architecture -sku $sku + Write-Host "Installing .NET Core $sku $Version..." cmd /c start /wait $Installer /install /passive /norestart if ($LASTEXITCODE -eq 3010) { Write-Verbose "Restart required" } elseif ($LASTEXITCODE -ne 0) { - throw "Failure to install .NET Core SDK" + throw "Failure to install .NET SDK" } } -$switches = @( - '-Architecture',$arch -) +$switches = @() $envVars = @{ # For locally installed dotnet, skip first time experience which takes a long time 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' = 'true'; @@ -161,15 +162,37 @@ if ($InstallLocality -eq 'machine') { $DotNetInstallDir = '/usr/share/dotnet' } else { $restartRequired = $false - if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { - Install-DotNet -Version $sdkVersion + if ($PSCmdlet.ShouldProcess(".NET SDK $sdkVersion", "Install")) { + Install-DotNet -Version $sdkVersion -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $sdkVersion -Architecture x86 + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } + } + + $runtimeVersions | Sort-Object | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET runtime $_", "Install")) { + Install-DotNet -Version $_ -sku Runtime -Architecture $arch + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $_ -sku Runtime -Architecture x86 + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } + } } - $runtimeVersions | Get-Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { - Install-DotNet -Version $_ -Runtime + $windowsDesktopRuntimeVersions | Sort-Object | Get-Unique |% { + if ($PSCmdlet.ShouldProcess(".NET Windows Desktop $_", "Install")) { + Install-DotNet -Version $_ -sku WindowsDesktop -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + + if ($IncludeX86) { + Install-DotNet -Version $_ -sku WindowsDesktop -Architecture x86 + $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) + } } } @@ -182,20 +205,34 @@ if ($InstallLocality -eq 'machine') { } } elseif ($InstallLocality -eq 'repo') { $DotNetInstallDir = "$DotNetInstallScriptRoot/.dotnet" + $DotNetX86InstallDir = "$DotNetInstallScriptRoot/x86/.dotnet" } elseif ($env:AGENT_TOOLSDIRECTORY) { $DotNetInstallDir = "$env:AGENT_TOOLSDIRECTORY/dotnet" + $DotNetX86InstallDir = "$env:AGENT_TOOLSDIRECTORY/x86/dotnet" } else { $DotNetInstallDir = Join-Path $HOME .dotnet } -Write-Host "Installing .NET Core SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue - if ($DotNetInstallDir) { - $switches += '-InstallDir',"`"$DotNetInstallDir`"" + if (!(Test-Path $DotNetInstallDir)) { New-Item -ItemType Directory -Path $DotNetInstallDir } + $DotNetInstallDir = Resolve-Path $DotNetInstallDir + Write-Host "Installing .NET SDK and runtimes to $DotNetInstallDir" -ForegroundColor Blue $envVars['DOTNET_MULTILEVEL_LOOKUP'] = '0' $envVars['DOTNET_ROOT'] = $DotNetInstallDir } +if ($IncludeX86) { + if ($DotNetX86InstallDir) { + if (!(Test-Path $DotNetX86InstallDir)) { New-Item -ItemType Directory -Path $DotNetX86InstallDir } + $DotNetX86InstallDir = Resolve-Path $DotNetX86InstallDir + Write-Host "Installing x86 .NET SDK and runtimes to $DotNetX86InstallDir" -ForegroundColor Blue + } else { + # Only machine-wide or repo-wide installations can handle two unique dotnet.exe architectures. + Write-Error "The installation location or OS isn't supported for x86 installation. Try a different -InstallLocality value." + return 1 + } +} + if ($IsMacOS -or $IsLinux) { $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/781752509a890ca7520f1182e8bae71f9a53d754/src/dotnet-install.sh" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" @@ -219,47 +256,89 @@ $DotNetInstallScriptPathExpression = "& '$DotNetInstallScriptPathExpression'" $anythingInstalled = $false $global:LASTEXITCODE = 0 -if ($PSCmdlet.ShouldProcess(".NET Core SDK $sdkVersion", "Install")) { +if ($PSCmdlet.ShouldProcess(".NET SDK $sdkVersion", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion $switches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture $arch -InstallDir $DotNetInstallDir $switches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion $switches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture $arch -InstallDir $DotNetInstallDir $switches -DryRun" +} + +if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET x86 SDK $sdkVersion", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture x86 -InstallDir $DotNetX86InstallDir $switches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET x86 SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Version $sdkVersion -Architecture x86 -InstallDir $DotNetX86InstallDir $switches -DryRun" + } } $dotnetRuntimeSwitches = $switches + '-Runtime','dotnet' $runtimeVersions | Sort-Object -Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core runtime $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET Core $Arch runtime $_", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $dotnetRuntimeSwitches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $dotnetRuntimeSwitches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $dotnetRuntimeSwitches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $dotnetRuntimeSwitches -DryRun" + } + + if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET Core x86 runtime $_", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $dotnetRuntimeSwitches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $dotnetRuntimeSwitches -DryRun" + } } } $windowsDesktopRuntimeSwitches = $switches + '-Runtime','windowsdesktop' $windowsDesktopRuntimeVersions | Sort-Object -Unique |% { - if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop runtime $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop $arch runtime $_", "Install")) { $anythingInstalled = $true - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $windowsDesktopRuntimeSwitches" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $windowsDesktopRuntimeSwitches" if ($LASTEXITCODE -ne 0) { Write-Error ".NET SDK installation failure: $LASTEXITCODE" exit $LASTEXITCODE } } else { - Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ $windowsDesktopRuntimeSwitches -DryRun" + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture $arch -InstallDir $DotNetInstallDir $windowsDesktopRuntimeSwitches -DryRun" + } + + if ($IncludeX86) { + if ($PSCmdlet.ShouldProcess(".NET Core WindowsDesktop x86 runtime $_", "Install")) { + $anythingInstalled = $true + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $windowsDesktopRuntimeSwitches" + + if ($LASTEXITCODE -ne 0) { + Write-Error ".NET SDK installation failure: $LASTEXITCODE" + exit $LASTEXITCODE + } + } else { + Invoke-Expression -Command "$DotNetInstallScriptPathExpression -Channel $_ -Architecture x86 -InstallDir $DotNetX86InstallDir $windowsDesktopRuntimeSwitches -DryRun" + } } } diff --git a/tools/Install-NuGetCredProvider.ps1 b/tools/Install-NuGetCredProvider.ps1 index 6d310034..496049a2 100755 --- a/tools/Install-NuGetCredProvider.ps1 +++ b/tools/Install-NuGetCredProvider.ps1 @@ -33,7 +33,7 @@ if ($IsMacOS -or $IsLinux) { $installerScript = Join-Path $toolsPath $installerScript -if (!(Test-Path $installerScript)) { +if (!(Test-Path $installerScript) -or $Force) { Invoke-WebRequest $sourceUrl -OutFile $installerScript } @@ -43,7 +43,7 @@ if ($IsMacOS -or $IsLinux) { chmod u+x $installerScript } -& $installerScript -Force:$Force +& $installerScript -Force:$Force -AddNetfx -InstallNet6 if ($AccessToken) { $endpoints = @()