Support for deployment in Azure Government Cloud (Leonardo)

http/src/main/resources/init-resources/azure_vm_init_script.sh

@@ -109,6 +109,8 @@ SAMURL=$6
 SAMRESOURCEID=$7
 CONTENTSECURITYPOLICY_FILE=$8
 
+RELAY_SUFFIX=${21:-".servicebus.windows.net"}
+
 # Envs for welder
 WELDER_WSM_URL=${9:-localhost}
 WORKSPACE_ID="${10:-dummy}" # Additionally used for welder
@@ -126,11 +128,11 @@ WORKSPACE_STORAGE_CONTAINER_URL="${17:-dummy}"
 SERVER_APP_BASE_URL="/${RELAY_CONNECTION_NAME}/"
 SERVER_APP_ALLOW_ORIGIN="*"
 HCVAR='\$hc'
-SERVER_APP_WEBSOCKET_URL="wss://${RELAY_NAME}.servicebus.windows.net/${HCVAR}/${RELAY_CONNECTION_NAME}"
-SERVER_APP_WEBSOCKET_HOST="${RELAY_NAME}.servicebus.windows.net"
+SERVER_APP_WEBSOCKET_URL="wss://${RELAY_NAME}${RELAY_SUFFIX}/${HCVAR}/${RELAY_CONNECTION_NAME}"
+SERVER_APP_WEBSOCKET_HOST="${RELAY_NAME}${RELAY_SUFFIX}"
 
 # Relay listener configuration
-RELAY_CONNECTIONSTRING="Endpoint=sb://${RELAY_NAME}.servicebus.windows.net/;SharedAccessKeyName=listener;SharedAccessKey=${RELAY_CONNECTION_POLICY_KEY};EntityPath=${RELAY_CONNECTION_NAME}"
+RELAY_CONNECTIONSTRING="Endpoint=sb://${RELAY_NAME}${RELAY_SUFFIX}/;SharedAccessKeyName=listener;SharedAccessKey=${RELAY_CONNECTION_POLICY_KEY};EntityPath=${RELAY_CONNECTION_NAME}"
 
 # Relay listener configuration - setDateAccessed listener
 LEONARDO_URL="${18:-dummy}"
@@ -143,6 +145,7 @@ echo "RELAY_NAME = ${RELAY_NAME}"
 echo "RELAY_CONNECTION_NAME = ${RELAY_CONNECTION_NAME}"
 echo "RELAY_TARGET_HOST = ${RELAY_TARGET_HOST}"
 echo "RELAY_CONNECTION_POLICY_KEY = ${RELAY_CONNECTION_POLICY_KEY}"
+echo "RELAY_SUFFIX = ${RELAY_SUFFIX}"
 echo "LISTENER_DOCKER_IMAGE = ${LISTENER_DOCKER_IMAGE}"
 echo "SAMURL = ${SAMURL}"
 echo "SAMRESOURCEID = ${SAMRESOURCEID}"

http/src/main/resources/leo.conf

@@ -162,7 +162,7 @@ azure {
       # If true, it is assumed that Leo is hosted on Azure and will use Azure managed identity for authentication.
       enabled = ${?AZURE_HOSTING_MODE_ENABLED}
       # valid values are AZURE (Azure Commercial), AZURE_US_GOVERNMENT and AZURE_CHINA
-      azure-environment = ${?AZURE_HOSTING_ENVIRONMENT}
+      azure-environment = ${?AZURE_ENVIRONMENT}
       managed-identity-auth-config{
         token-scope = ${?AZURE_MI_TOKEN_SCOPE}
         token-acquisition-timeout = ${?AZURE_MI_TOKEN_ACQUISITION_TIMEOUT}

http/src/main/resources/reference.conf

@@ -255,7 +255,7 @@ azure {
         type = "CustomScript",
         version = "2.1",
         minor-version-auto-upgrade = true,
-        file-uris = ["https://raw.githubusercontent.com/DataBiosphere/leonardo/8390d25ccd761fb206cf388560a571be77a42bbd/http/src/main/resources/init-resources/azure_vm_init_script.sh"]
+        file-uris = ["https://raw.githubusercontent.com/DataBiosphere/leonardo/f58c237b4dc235cd1c24c6dfc7500c07bdbd5bc3/http/src/main/resources/init-resources/azure_vm_init_script.sh"]
       }
       # [IA-4997] to support CHIPS by setting partitioned cookies
       # listener-image = "terradevacrpublic.azurecr.io/terra-azure-relay-listeners:474f157"

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/CromwellAppInstall.scala

@@ -7,7 +7,8 @@ import org.broadinstitute.dsde.workbench.azure.{AzureApplicationInsightsService,
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.{AppContext, WsmControlledDatabaseResource}
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.CoaAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, CoaAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -25,7 +26,8 @@ class CromwellAppInstall[F[_]](config: CoaAppConfig,
                                cromwellDao: CromwellDAO[F],
                                cbasDao: CbasDAO[F],
                                azureBatchService: AzureBatchService[F],
-                               azureApplicationInsightsService: AzureApplicationInsightsService[F]
+                               azureApplicationInsightsService: AzureApplicationInsightsService[F],
+                               authProvider: SamAuthProvider[F]
 )(implicit
   F: Async[F]
 ) extends AppInstall[F] {
@@ -69,10 +71,7 @@ class CromwellAppInstall[F[_]](config: CoaAppConfig,
 
     // Get the pet userToken
     tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-    userToken <- F.fromOption(
-      tokenOpt,
-      AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-    )
+    userToken <- F.pure(tokenOpt.getOrElse("")) // Empty token when running on Azure.
 
     values = List(
       // azure resources configs
@@ -85,13 +84,22 @@ class CromwellAppInstall[F[_]](config: CoaAppConfig,
       raw"config.subscriptionId=${params.cloudContext.subscriptionId.value}",
       raw"config.region=${params.landingZoneResources.region}",
       raw"config.applicationInsightsConnectionString=${applicationInsightsComponent.connectionString()}",
+      raw"config.azureEnvironment=${ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment}",
+      raw"config.azureManagementTokenScope=${AzureEnvironmentConverter
+          .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+          .getResourceManagerEndpoint}.default",
+      raw"config.batchAccountSuffix=${AzureEnvironmentConverter
+          .batchAccountSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
 
       // relay configs
       raw"relay.path=${params.relayPath.renderString}",
 
       // persistence configs
       raw"persistence.storageResourceGroup=${params.cloudContext.managedResourceGroupName.value}",
       raw"persistence.storageAccount=${params.landingZoneResources.storageAccountName.value}",
+      raw"persistence.storageAccountSuffix=${AzureEnvironmentConverter
+          .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+          .getStorageEndpointSuffix}",
       raw"persistence.blobContainer=${storageContainer.name.value}",
       raw"persistence.leoAppInstanceName=${params.app.appName.value}",
       raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
@@ -124,7 +132,8 @@ class CromwellAppInstall[F[_]](config: CoaAppConfig,
 
       // Database configs
       raw"postgres.podLocalDatabaseEnabled=false",
-      raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+      raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+          .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
       raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
       // convention is that the database user is the same as the service account name
       raw"postgres.user=${params.ksaName.value}",

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/CromwellRunnerAppInstall.scala

@@ -9,7 +9,7 @@ import org.broadinstitute.dsde.workbench.leonardo.{AppContext, WsmControlledData
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.{ControlledDatabase, ReferenceDatabase}
 import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
-import org.broadinstitute.dsde.workbench.leonardo.config.{CromwellRunnerAppConfig, SamConfig}
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, CromwellRunnerAppConfig, SamConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao.{BpmApiClientProvider, CromwellDAO, SamDAO}
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -80,11 +80,6 @@ class CromwellRunnerAppInstall[F[_]](config: CromwellRunnerAppConfig,
       )
 
       // Get the pet userToken
-      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
 
       leoAuth <- authProvider.getLeoAuthToken
 
@@ -99,6 +94,10 @@ class CromwellRunnerAppInstall[F[_]](config: CromwellRunnerAppConfig,
           .map(v => raw"config.concurrentJobLimit=${v}")
       }
 
+      // Get the pet userToken
+      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
+      userToken <- F.pure(tokenOpt.getOrElse("")) // Empty token when running on Azure.
+
       values = List(
         // azure resources configs
         raw"config.resourceGroup=${params.cloudContext.managedResourceGroupName.value}",
@@ -110,12 +109,21 @@ class CromwellRunnerAppInstall[F[_]](config: CromwellRunnerAppConfig,
         raw"config.subscriptionId=${params.cloudContext.subscriptionId.value}",
         raw"config.region=${params.landingZoneResources.region}",
         raw"config.applicationInsightsConnectionString=${applicationInsightsComponent.connectionString()}",
+        raw"config.azureEnvironment=${ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment}",
+        raw"config.azureManagementTokenScope=${AzureEnvironmentConverter
+            .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+            .getResourceManagerEndpoint}.default",
+        raw"config.batchAccountSuffix=${AzureEnvironmentConverter
+            .batchAccountSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
 
         // relay configs
         raw"relay.path=${params.relayPath.renderString}",
 
         // persistence configs
         raw"persistence.storageAccount=${params.landingZoneResources.storageAccountName.value}",
+        raw"persistence.storageAccountSuffix=${AzureEnvironmentConverter
+            .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+            .getStorageEndpointSuffix}",
         raw"persistence.blobContainer=${storageContainer.name.value}",
         raw"persistence.leoAppInstanceName=${params.app.appName.value}",
         raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
@@ -138,7 +146,8 @@ class CromwellRunnerAppInstall[F[_]](config: CromwellRunnerAppConfig,
 
         // database configs
         raw"postgres.podLocalDatabaseEnabled=false",
-        raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+        raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+            .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
         raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
         // convention is that the database user is the same as the service account name
         raw"postgres.user=${params.ksaName.value}",
@@ -155,7 +164,10 @@ class CromwellRunnerAppInstall[F[_]](config: CromwellRunnerAppConfig,
 
         // Bard configs
         raw"bard.bardUrl=${config.bardBaseUri}",
-        raw"bard.enabled=${config.bardEnabled}"
+        raw"bard.enabled=${config.bardEnabled}",
+
+        // TEMPORARY HELM OVERRIDE VALUES WHILE WAITING FOR PR
+        raw"cromwell.image=potomacdevap.azurecr.us/broadinstitute/cromwell:84214c9d71721269f9945406d72e30a6f8aac514"
       )
 
       finalList = maybeLimits match {

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/HailBatchAppInstall.scala

@@ -3,8 +3,9 @@ import cats.effect.Async
 import cats.mtl.Ask
 import cats.syntax.all._
 import org.broadinstitute.dsde.workbench.leonardo.AppContext
-import org.broadinstitute.dsde.workbench.leonardo.config.HailBatchAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, HailBatchAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao.HailBatchDAO
+import org.broadinstitute.dsde.workbench.leonardo.http.ConfigReader
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
 import org.broadinstitute.dsp.Values
 import org.http4s.Uri
@@ -34,7 +35,9 @@ class HailBatchAppInstall[F[_]](config: HailBatchAppConfig, hailBatchDao: HailBa
           raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
           raw"persistence.workspaceManager.workspaceId=${params.workspaceId.value}",
           raw"persistence.workspaceManager.containerResourceId=${storageContainer.resourceId.value.toString}",
-          raw"persistence.workspaceManager.storageContainerUrl=https://${params.landingZoneResources.storageAccountName.value}.blob.core.windows.net/${storageContainer.name.value}",
+          raw"persistence.workspaceManager.storageContainerUrl=https://${params.landingZoneResources.storageAccountName.value}.blob${AzureEnvironmentConverter
+              .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+              .getStorageEndpointSuffix}/${storageContainer.name.value}",
           raw"persistence.leoAppName=${params.app.appName.value}",
 
           // identity configs

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/WdsAppInstall.scala

@@ -7,7 +7,8 @@ import cats.mtl.Ask
 import cats.syntax.all._
 import org.broadinstitute.dsde.workbench.azure.AzureApplicationInsightsService
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.WdsAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, WdsAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -23,7 +24,8 @@ class WdsAppInstall[F[_]](config: WdsAppConfig,
                           tdrConfig: TdrConfig,
                           samDao: SamDAO[F],
                           wdsDao: WdsDAO[F],
-                          azureApplicationInsightsService: AzureApplicationInsightsService[F]
+                          azureApplicationInsightsService: AzureApplicationInsightsService[F],
+                          authProvider: SamAuthProvider[F]
 )(implicit
   F: Async[F]
 ) extends AppInstall[F] {
@@ -54,15 +56,14 @@ class WdsAppInstall[F[_]](config: WdsAppConfig,
       )
 
       // Get the pet userToken
-      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
 
       // Get Vpa enabled tag
       vpaEnabled <- F.pure(params.landingZoneResources.aksCluster.tags.getOrElse("aks-cost-vpa-enabled", false))
 
+      // Get the pet userToken
+      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
+      userToken <- F.pure(tokenOpt.getOrElse("")) // Empty token when running on Azure.
+
       valuesList =
         List(
           // pass enviiroment information to wds so it can properly pick its config
@@ -96,7 +97,8 @@ class WdsAppInstall[F[_]](config: WdsAppConfig,
           raw"provenance.sourceWorkspaceId=${params.app.sourceWorkspaceId.map(_.value).getOrElse("")}",
 
           // database configs
-          raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+          raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+              .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
           raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
           raw"postgres.dbname=$dbName",
           // convention is that the database user is the same as the service account name

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/WorkflowsAppInstall.scala

@@ -8,7 +8,7 @@ import org.broadinstitute.dsde.workbench.azure.{AzureApplicationInsightsService,
 import org.broadinstitute.dsde.workbench.leonardo.{AppContext, WsmControlledDatabaseResource}
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.WorkflowsAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, WorkflowsAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -69,10 +69,7 @@ class WorkflowsAppInstall[F[_]](config: WorkflowsAppConfig,
 
       // Get the pet userToken
       tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
+      userToken <- F.pure(tokenOpt.getOrElse("")) // Empty token when running on Azure.
 
       values =
         List(
@@ -111,7 +108,8 @@ class WorkflowsAppInstall[F[_]](config: WorkflowsAppConfig,
 
           // database configs
           raw"postgres.podLocalDatabaseEnabled=false",
-          raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+          raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+              .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
           raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
           // convention is that the database user is the same as the service account name
           raw"postgres.user=${params.ksaName.value}",

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/AzureHostingModeConfig.scala

@@ -17,15 +17,46 @@
 )
 
 object AzureEnvironmentConverter {
-  val Azure: String = "AZURE"
-  val AzureGov: String = "AZURE_US_GOVERNMENT"
-  val AzureChina: String = "AZURE_CHINA"
+  val Azure: String = "AzureCloud"
+  val AzureGov: String = "AzureUSGovernmentCloud"
 
   def fromString(s: String): AzureEnvironment = s match {
-    case AzureGov   => AzureEnvironment.AZURE_US_GOVERNMENT
-    case AzureChina => AzureEnvironment.AZURE_CHINA
+    case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT
     // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
     case Azure => AzureEnvironment.AZURE
     case _     => AzureEnvironment.AZURE
   }
+
+  // servicebus suffix not currently provided by AzureEnvironment library, values found here
+  def relaySuffixFromEnvironment(azureEnvironment: AzureEnvironment): String = azureEnvironment match {
+    case AzureEnvironment.AZURE_US_GOVERNMENT => ".servicebus.usgovcloudapi.net"
+    // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
+    case AzureEnvironment.AZURE => ".servicebus.windows.net"
+    case _                      => ".servicebus.windows.net"
+  }
+
+  def relaySuffixFromString(s: String): String =
+    relaySuffixFromEnvironment(fromString(s))
+
+  // database suffix not currently provided by AzureEnvironment library, values found here
+  def postgresSuffixFromEnvironment(azureEnvironment: AzureEnvironment): String = azureEnvironment match {
+    case AzureEnvironment.AZURE_US_GOVERNMENT => ".database.usgovcloudapi.net"
+    // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
+    case AzureEnvironment.AZURE => ".database.azure.com"
+    case _                      => ".database.azure.com"
+  }
+
+  def postgresSuffixFromString(s: String): String =
+    postgresSuffixFromEnvironment(fromString(s))
+
+  // batchAccount suffix not currently provided by AzureEnvironment library, values found here
+  def batchAccountSuffixFromEnvironment(azureEnvironment: AzureEnvironment): String = azureEnvironment match {
+    case AzureEnvironment.AZURE_US_GOVERNMENT => ".batch.usgovcloudapi.net"
+    // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
+    case AzureEnvironment.AZURE => ".batch.azure.com"
+    case _                      => ".batch.azure.com"
+  }
+
+  def batchAccountSuffixFromString(s: String): String =
+    batchAccountSuffixFromEnvironment(fromString(s))
 }