Support for deployment in Azure Government Cloud (Leonardo)

http/src/main/resources/init-resources/azure_vm_init_script.sh

@@ -109,6 +109,8 @@ SAMURL=$6
 SAMRESOURCEID=$7
 CONTENTSECURITYPOLICY_FILE=$8
 
+RELAY_SUFFIX=${21:-".servicebus.windows.net"}
+
 # Envs for welder
 WELDER_WSM_URL=${9:-localhost}
 WORKSPACE_ID="${10:-dummy}" # Additionally used for welder
@@ -126,11 +128,11 @@ WORKSPACE_STORAGE_CONTAINER_URL="${17:-dummy}"
 SERVER_APP_BASE_URL="/${RELAY_CONNECTION_NAME}/"
 SERVER_APP_ALLOW_ORIGIN="*"
 HCVAR='\$hc'
-SERVER_APP_WEBSOCKET_URL="wss://${RELAY_NAME}.servicebus.windows.net/${HCVAR}/${RELAY_CONNECTION_NAME}"
-SERVER_APP_WEBSOCKET_HOST="${RELAY_NAME}.servicebus.windows.net"
+SERVER_APP_WEBSOCKET_URL="wss://${RELAY_NAME}${RELAY_SUFFIX}/${HCVAR}/${RELAY_CONNECTION_NAME}"
+SERVER_APP_WEBSOCKET_HOST="${RELAY_NAME}${RELAY_SUFFIX}"
 
 # Relay listener configuration
-RELAY_CONNECTIONSTRING="Endpoint=sb://${RELAY_NAME}.servicebus.windows.net/;SharedAccessKeyName=listener;SharedAccessKey=${RELAY_CONNECTION_POLICY_KEY};EntityPath=${RELAY_CONNECTION_NAME}"
+RELAY_CONNECTIONSTRING="Endpoint=sb://${RELAY_NAME}${RELAY_SUFFIX}/;SharedAccessKeyName=listener;SharedAccessKey=${RELAY_CONNECTION_POLICY_KEY};EntityPath=${RELAY_CONNECTION_NAME}"
 
 # Relay listener configuration - setDateAccessed listener
 LEONARDO_URL="${18:-dummy}"
@@ -143,6 +145,7 @@ echo "RELAY_NAME = ${RELAY_NAME}"
 echo "RELAY_CONNECTION_NAME = ${RELAY_CONNECTION_NAME}"
 echo "RELAY_TARGET_HOST = ${RELAY_TARGET_HOST}"
 echo "RELAY_CONNECTION_POLICY_KEY = ${RELAY_CONNECTION_POLICY_KEY}"
+echo "RELAY_SUFFIX = ${RELAY_SUFFIX}"
 echo "LISTENER_DOCKER_IMAGE = ${LISTENER_DOCKER_IMAGE}"
 echo "SAMURL = ${SAMURL}"
 echo "SAMRESOURCEID = ${SAMRESOURCEID}"

http/src/main/resources/leo.conf

@@ -165,7 +165,7 @@ azure {
       # If true, it is assumed that Leo is hosted on Azure and will use Azure managed identity for authentication.
       enabled = ${?AZURE_HOSTING_MODE_ENABLED}
       # valid values are AZURE (Azure Commercial), AZURE_US_GOVERNMENT and AZURE_CHINA
-      azure-environment = ${?AZURE_HOSTING_ENVIRONMENT}
+      azure-environment = ${?AZURE_ENVIRONMENT}
       managed-identity-auth-config{
         token-scope = ${?AZURE_MI_TOKEN_SCOPE}
         token-acquisition-timeout = ${?AZURE_MI_TOKEN_ACQUISITION_TIMEOUT}

http/src/main/resources/reference.conf

@@ -255,7 +255,7 @@ azure {
         type = "CustomScript",
         version = "2.1",
         minor-version-auto-upgrade = true,
-        file-uris = ["https://raw.githubusercontent.com/DataBiosphere/leonardo/8390d25ccd761fb206cf388560a571be77a42bbd/http/src/main/resources/init-resources/azure_vm_init_script.sh"]
+        file-uris = ["https://raw.githubusercontent.com/DataBiosphere/leonardo/f58c237b4dc235cd1c24c6dfc7500c07bdbd5bc3/http/src/main/resources/init-resources/azure_vm_init_script.sh"]
       }
       # [IA-4997] to support CHIPS by setting partitioned cookies
       # listener-image = "terradevacrpublic.azurecr.io/terra-azure-relay-listeners:474f157"
@@ -374,7 +374,7 @@ azure {
   cromwell-runner-app-config {
     instrumentation-enabled = false
     chart-name = "terra-helm/cromwell-runner-app"
-    chart-version = "0.197.0"
+    chart-version = "0.198.0"
     release-name-suffix = "cra-rls"
     namespace-name-suffix = "cra-ns"
     ksa-name = "cra-ksa"

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/CromwellAppInstall.scala

@@ -7,7 +7,8 @@
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.{AppContext, WsmControlledDatabaseResource}
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.CoaAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, CoaAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -25,7 +26,8 @@
                                cromwellDao: CromwellDAO[F],
                                cbasDao: CbasDAO[F],
                                azureBatchService: AzureBatchService[F],
-                               azureApplicationInsightsService: AzureApplicationInsightsService[F]
+                               azureApplicationInsightsService: AzureApplicationInsightsService[F],
+                               authProvider: SamAuthProvider[F]
 )(implicit
   F: Async[F]
 ) extends AppInstall[F] {
@@ -69,10 +71,15 @@
 
     // Get the pet userToken
     tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-    userToken <- F.fromOption(
-      tokenOpt,
-      AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-    )
+    userToken <- ConfigReader.appConfig.azure.hostingModeConfig.enabled match {
+      case false =>
+        F.fromOption(
+          tokenOpt,
+          AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
+        )
+      case true =>
+        F.pure("") // No pet user token in Azure.
+    }
 
     values = List(
       // azure resources configs
@@ -85,13 +92,22 @@
       raw"config.subscriptionId=${params.cloudContext.subscriptionId.value}",
       raw"config.region=${params.landingZoneResources.region}",
       raw"config.applicationInsightsConnectionString=${applicationInsightsComponent.connectionString()}",
+      raw"config.azureEnvironment=${ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment}",
+      raw"config.azureManagementTokenScope=${AzureEnvironmentConverter
+          .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+          .getResourceManagerEndpoint}.default",
+      raw"config.batchAccountSuffix=${AzureEnvironmentConverter
+          .batchAccountSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
 
       // relay configs
       raw"relay.path=${params.relayPath.renderString}",
 
       // persistence configs
       raw"persistence.storageResourceGroup=${params.cloudContext.managedResourceGroupName.value}",
       raw"persistence.storageAccount=${params.landingZoneResources.storageAccountName.value}",
+      raw"persistence.storageAccountSuffix=${AzureEnvironmentConverter
+          .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+          .getStorageEndpointSuffix}",
       raw"persistence.blobContainer=${storageContainer.name.value}",
       raw"persistence.leoAppInstanceName=${params.app.appName.value}",
       raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
@@ -124,7 +140,8 @@
 
       // Database configs
       raw"postgres.podLocalDatabaseEnabled=false",
-      raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+      raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+          .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
       raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
       // convention is that the database user is the same as the service account name
       raw"postgres.user=${params.ksaName.value}",

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/CromwellRunnerAppInstall.scala

@@ -9,7 +9,7 @@
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.{ControlledDatabase, ReferenceDatabase}
 import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
-import org.broadinstitute.dsde.workbench.leonardo.config.{CromwellRunnerAppConfig, SamConfig}
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, CromwellRunnerAppConfig, SamConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao.{BpmApiClientProvider, CromwellDAO, SamDAO}
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -80,11 +80,6 @@
       )
 
       // Get the pet userToken
-      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
 
       leoAuth <- authProvider.getLeoAuthToken
 
@@ -99,6 +94,18 @@
           .map(v => raw"config.concurrentJobLimit=${v}")
       }
 
+      // Get the pet userToken
+      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
+      userToken <- ConfigReader.appConfig.azure.hostingModeConfig.enabled match {
+        case false =>
+          F.fromOption(
+            tokenOpt,
+            AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
+          )
+        case true =>
+          F.pure("") // No pet user token in Azure.
+      }
+
       values = List(
         // azure resources configs
         raw"config.resourceGroup=${params.cloudContext.managedResourceGroupName.value}",
@@ -110,12 +117,21 @@
         raw"config.subscriptionId=${params.cloudContext.subscriptionId.value}",
         raw"config.region=${params.landingZoneResources.region}",
         raw"config.applicationInsightsConnectionString=${applicationInsightsComponent.connectionString()}",
+        raw"config.azureEnvironment=${ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment}",
+        raw"config.azureManagementTokenScope=${AzureEnvironmentConverter
+            .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+            .getResourceManagerEndpoint}.default",
+        raw"config.batchAccountSuffix=${AzureEnvironmentConverter
+            .batchAccountSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
 
         // relay configs
         raw"relay.path=${params.relayPath.renderString}",
 
         // persistence configs
         raw"persistence.storageAccount=${params.landingZoneResources.storageAccountName.value}",
+        raw"persistence.storageAccountSuffix=${AzureEnvironmentConverter
+            .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+            .getStorageEndpointSuffix}",
         raw"persistence.blobContainer=${storageContainer.name.value}",
         raw"persistence.leoAppInstanceName=${params.app.appName.value}",
         raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
@@ -138,7 +154,8 @@
 
         // database configs
         raw"postgres.podLocalDatabaseEnabled=false",
-        raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+        raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+            .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
         raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
         // convention is that the database user is the same as the service account name
         raw"postgres.user=${params.ksaName.value}",

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/HailBatchAppInstall.scala

@@ -3,8 +3,9 @@ import cats.effect.Async
 import cats.mtl.Ask
 import cats.syntax.all._
 import org.broadinstitute.dsde.workbench.leonardo.AppContext
-import org.broadinstitute.dsde.workbench.leonardo.config.HailBatchAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, HailBatchAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao.HailBatchDAO
+import org.broadinstitute.dsde.workbench.leonardo.http.ConfigReader
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
 import org.broadinstitute.dsp.Values
 import org.http4s.Uri
@@ -34,7 +35,9 @@ class HailBatchAppInstall[F[_]](config: HailBatchAppConfig, hailBatchDao: HailBa
           raw"persistence.workspaceManager.url=${params.config.wsmConfig.uri.renderString}",
           raw"persistence.workspaceManager.workspaceId=${params.workspaceId.value}",
           raw"persistence.workspaceManager.containerResourceId=${storageContainer.resourceId.value.toString}",
-          raw"persistence.workspaceManager.storageContainerUrl=https://${params.landingZoneResources.storageAccountName.value}.blob.core.windows.net/${storageContainer.name.value}",
+          raw"persistence.workspaceManager.storageContainerUrl=https://${params.landingZoneResources.storageAccountName.value}.blob${AzureEnvironmentConverter
+              .fromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)
+              .getStorageEndpointSuffix}/${storageContainer.name.value}",
           raw"persistence.leoAppName=${params.app.appName.value}",
 
           // identity configs

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/WdsAppInstall.scala

@@ -7,7 +7,8 @@
 import cats.syntax.all._
 import org.broadinstitute.dsde.workbench.azure.AzureApplicationInsightsService
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.WdsAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.auth.SamAuthProvider
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, WdsAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -23,7 +24,8 @@
                           tdrConfig: TdrConfig,
                           samDao: SamDAO[F],
                           wdsDao: WdsDAO[F],
-                          azureApplicationInsightsService: AzureApplicationInsightsService[F]
+                          azureApplicationInsightsService: AzureApplicationInsightsService[F],
+                          authProvider: SamAuthProvider[F]
 )(implicit
   F: Async[F]
 ) extends AppInstall[F] {
@@ -54,15 +56,22 @@
       )
 
       // Get the pet userToken
-      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
 
       // Get Vpa enabled tag
       vpaEnabled <- F.pure(params.landingZoneResources.aksCluster.tags.getOrElse("aks-cost-vpa-enabled", false))
 
+      // Get the pet userToken
+      tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
+      userToken <- ConfigReader.appConfig.azure.hostingModeConfig.enabled match {
+        case false =>
+          F.fromOption(
+            tokenOpt,
+            AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
+          )
+        case true =>
+          F.pure("") // No pet user token in Azure.
+      }
+
       valuesList =
         List(
           // pass enviiroment information to wds so it can properly pick its config
@@ -96,7 +105,8 @@
           raw"provenance.sourceWorkspaceId=${params.app.sourceWorkspaceId.map(_.value).getOrElse("")}",
 
           // database configs
-          raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+          raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+              .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
           raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
           raw"postgres.dbname=$dbName",
           // convention is that the database user is the same as the service account name

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/app/WorkflowsAppInstall.scala

@@ -8,7 +8,7 @@
 import org.broadinstitute.dsde.workbench.leonardo.{AppContext, WsmControlledDatabaseResource}
 import org.broadinstitute.dsde.workbench.leonardo.app.AppInstall.getAzureDatabaseName
 import org.broadinstitute.dsde.workbench.leonardo.app.Database.ControlledDatabase
-import org.broadinstitute.dsde.workbench.leonardo.config.WorkflowsAppConfig
+import org.broadinstitute.dsde.workbench.leonardo.config.{AzureEnvironmentConverter, WorkflowsAppConfig}
 import org.broadinstitute.dsde.workbench.leonardo.dao._
 import org.broadinstitute.dsde.workbench.leonardo.http._
 import org.broadinstitute.dsde.workbench.leonardo.util.AppCreationException
@@ -69,10 +69,15 @@
 
       // Get the pet userToken
       tokenOpt <- samDao.getCachedArbitraryPetAccessToken(params.app.auditInfo.creator)
-      userToken <- F.fromOption(
-        tokenOpt,
-        AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
-      )
+      userToken <- ConfigReader.appConfig.azure.hostingModeConfig.enabled match {
+        case false =>
+          F.fromOption(
+            tokenOpt,
+            AppCreationException(s"Pet not found for user ${params.app.auditInfo.creator}", Some(ctx.traceId))
+          )
+        case true =>
+          F.pure("") // No pet user token in Azure.
+      }
 
       values =
         List(
@@ -111,7 +116,8 @@
 
           // database configs
           raw"postgres.podLocalDatabaseEnabled=false",
-          raw"postgres.host=${postgresServer.name}.postgres.database.azure.com",
+          raw"postgres.host=${postgresServer.name}.postgres${AzureEnvironmentConverter
+              .postgresSuffixFromString(ConfigReader.appConfig.azure.hostingModeConfig.azureEnvironment)}",
           raw"postgres.pgbouncer.enabled=${postgresServer.pgBouncerEnabled}",
           // convention is that the database user is the same as the service account name
           raw"postgres.user=${params.ksaName.value}",