diff --git a/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/request/image/ImageSettingsV4Request.java b/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/request/image/ImageSettingsV4Request.java index 14b9e7b2c91..44aa50a996e 100644 --- a/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/request/image/ImageSettingsV4Request.java +++ b/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/request/image/ImageSettingsV4Request.java @@ -20,7 +20,7 @@ public class ImageSettingsV4Request implements JsonEntity { @ApiModelProperty(StackModelDescription.IMAGE_ID) private String id; - @ApiModelProperty(StackModelDescription.OS_TYPE) + @ApiModelProperty(StackModelDescription.IMAGE_OS) private String os; public String getCatalog() { diff --git a/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/response/image/StackImageV4Response.java b/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/response/image/StackImageV4Response.java index 2285e9ec7be..e6f62ff8e53 100644 --- a/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/response/image/StackImageV4Response.java +++ b/core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/response/image/StackImageV4Response.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.sequenceiq.cloudbreak.doc.ModelDescriptions; import com.sequenceiq.cloudbreak.doc.ModelDescriptions.ImageModelDescription; import com.sequenceiq.common.model.JsonEntity; @@ -25,6 +26,9 @@ public class StackImageV4Response implements JsonEntity { @ApiModelProperty(ImageModelDescription.IMAGE_CATALOG_NAME) private String catalogName; + @ApiModelProperty(ModelDescriptions.StackModelDescription.IMAGE_OS) + private String os; + public String getName() { return name; } @@ -56,4 +60,12 @@ public String getCatalogName() { public void setCatalogName(String catalogName) { this.catalogName = catalogName; } + + public String getOs() { + return os; + } + + public void setOs(String os) { + this.os = os; + } } diff --git a/core-api/src/main/java/com/sequenceiq/cloudbreak/doc/ModelDescriptions.java b/core-api/src/main/java/com/sequenceiq/cloudbreak/doc/ModelDescriptions.java index cd9e5c52dad..1e98e81682f 100644 --- a/core-api/src/main/java/com/sequenceiq/cloudbreak/doc/ModelDescriptions.java +++ b/core-api/src/main/java/com/sequenceiq/cloudbreak/doc/ModelDescriptions.java @@ -219,7 +219,7 @@ public static class StackModelDescription { public static final String IMAGE_SETTINGS = "settings for custom images"; public static final String IMAGE_CATALOG = "custom image catalog URL"; public static final String IMAGE_ID = "virtual machine image id from ImageCatalog, machines of the cluster will be started from this image"; - public static final String OS_TYPE = "os type of the image, this property is only considered when no specific image id is provided"; + public static final String IMAGE_OS = "os of the image, this property is only considered when no specific image id is provided"; public static final String INSTANCE_GROUP_ADJUSTMENT = "instance group adjustment"; public static final String TAGS = "stack related tags"; public static final String APPLICATION_TAGS = "stack related application tags"; diff --git a/core-api/src/main/java/com/sequenceiq/distrox/api/v1/distrox/model/image/DistroXImageV1Request.java b/core-api/src/main/java/com/sequenceiq/distrox/api/v1/distrox/model/image/DistroXImageV1Request.java index a8d240860da..149b36a1a22 100644 --- a/core-api/src/main/java/com/sequenceiq/distrox/api/v1/distrox/model/image/DistroXImageV1Request.java +++ b/core-api/src/main/java/com/sequenceiq/distrox/api/v1/distrox/model/image/DistroXImageV1Request.java @@ -21,6 +21,9 @@ public class DistroXImageV1Request implements Serializable { @ApiModelProperty(StackModelDescription.IMAGE_ID) private String id; + @ApiModelProperty(StackModelDescription.IMAGE_OS) + private String os; + public String getCatalog() { return catalog; } @@ -36,4 +39,12 @@ public String getId() { public void setId(String id) { this.id = id; } + + public String getOs() { + return os; + } + + public void setOs(String os) { + this.os = os; + } } diff --git a/core/src/main/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogService.java b/core/src/main/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogService.java index 299bf189352..3884a7b39ef 100644 --- a/core/src/main/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogService.java +++ b/core/src/main/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogService.java @@ -291,7 +291,7 @@ public StatedImage getLatestBaseImageDefaultPreferred(ImageFilter imageFilter, P List baseImages = statedImages.getImages().getBaseImages(); Optional defaultBaseImage = getLatestImageDefaultPreferred(baseImages); if (defaultBaseImage.isEmpty()) { - throw new CloudbreakImageNotFoundException(baseImageNotFoundErrorMessage(platform, imageFilter.getImageCatalog())); + throw new CloudbreakImageNotFoundException(baseImageNotFoundErrorMessage(platform, imageFilter)); } return statedImage(defaultBaseImage.get(), statedImages.getImageCatalogUrl(), statedImages.getImageCatalogName()); } @@ -305,8 +305,7 @@ public StatedImage getImagePrewarmedDefaultPreferred(ImageFilter imageFilter, Pr .map(ImageCatalogPlatform::nameToLowerCase) .findFirst(); String platform = firstImage.orElse(""); - throw new CloudbreakImageNotFoundException( - imageNotFoundErrorMessage(platform, imageFilter.getClusterVersion(), imageFilter.getImageCatalog())); + throw new CloudbreakImageNotFoundException(imageNotFoundErrorMessage(platform, imageFilter)); } return statedImage(selectedImage.get(), statedImages.getImageCatalogUrl(), statedImages.getImageCatalogName()); } @@ -330,14 +329,22 @@ private List filterImagesByRuntimeVersion(String clusterVersion, List isSdxAvailable() { diff --git a/core/src/test/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogServiceDefaultNotFoundTest.java b/core/src/test/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogServiceDefaultNotFoundTest.java index c07a6df960c..45255800e78 100644 --- a/core/src/test/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogServiceDefaultNotFoundTest.java +++ b/core/src/test/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogServiceDefaultNotFoundTest.java @@ -111,7 +111,8 @@ public void testGetDefaultImageShouldThrowNotFoundException() throws Exception { try { underTest.getImagePrewarmedDefaultPreferred(imageFilter, image -> true); } catch (CloudbreakImageNotFoundException exception) { - Assertions.assertEquals("Could not find any image for platform 'gcp', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.", + Assertions.assertEquals( + "Could not find any image for platform 'gcp', os 'notimportant', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.", exception.getMessage()); } verify(providerSpecificImageFilter, never()).filterImages(any(), anyList()); @@ -126,11 +127,13 @@ public void testGetDefaultImageShouldThrowNotFoundException2() throws Exception when(cloudbreakVersionListProvider.getVersions(any())).thenReturn(catalog.getVersions().getCloudbreakVersions()); when(imageCatalog.getImageCatalogUrl()).thenReturn(DEFAULT_CDH_IMAGE_CATALOG); - ImageFilter imageFilter = new ImageFilter(imageCatalog, Set.of(imageCatalogPlatform("aws")), "2.6", true, Set.of("centos7", "amazonlinux2"), null); + ImageFilter imageFilter = new ImageFilter(imageCatalog, Set.of(imageCatalogPlatform("aws")), "2.6", true, Set.of("centos7"), null); try { underTest.getImagePrewarmedDefaultPreferred(imageFilter, image -> true); } catch (CloudbreakImageNotFoundException exception) { - Assertions.assertEquals("Could not find any image for platform 'aws', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.", + Assertions.assertEquals( + "Could not find any image for platform 'aws', os 'centos7', runtime 'null' " + + "and Cloudbreak version '5.0.0' in 'null' image catalog.", exception.getMessage()); } verify(providerSpecificImageFilter, times(3)).filterImages(eq(Set.of(imageCatalogPlatform(PROVIDERS[0]))), anyList()); diff --git a/core/src/test/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverterTest.java b/core/src/test/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverterTest.java index ead67cb51ab..71f5362f09f 100644 --- a/core/src/test/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverterTest.java +++ b/core/src/test/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverterTest.java @@ -23,12 +23,14 @@ void testConvertDistroXImageV1RequestToImageSettingsV4Request() { DistroXImageV1Request input = new DistroXImageV1Request(); input.setCatalog("someCatalog"); input.setId("someId"); + input.setOs("someOs"); ImageSettingsV4Request result = underTest.convert(input); assertNotNull(result); assertEquals(input.getCatalog(), result.getCatalog()); assertEquals(input.getId(), result.getId()); + assertEquals(input.getOs(), result.getOs()); } @Test @@ -36,12 +38,14 @@ void testConvertImageSettingsV4RequestToDistroXImageV1Request() { ImageSettingsV4Request input = new ImageSettingsV4Request(); input.setCatalog("someCatalog"); input.setId("someId"); + input.setOs("someOs"); DistroXImageV1Request result = underTest.convert(input); assertNotNull(result); assertEquals(input.getCatalog(), result.getCatalog()); assertEquals(input.getId(), result.getId()); + assertEquals(input.getOs(), result.getOs()); } } \ No newline at end of file diff --git a/core/src/test/java/com/sequenceiq/distrox/v1/distrox/service/DistroXServiceTest.java b/core/src/test/java/com/sequenceiq/distrox/v1/distrox/service/DistroXServiceTest.java index 891b3c88498..7d8a5fbe48f 100644 --- a/core/src/test/java/com/sequenceiq/distrox/v1/distrox/service/DistroXServiceTest.java +++ b/core/src/test/java/com/sequenceiq/distrox/v1/distrox/service/DistroXServiceTest.java @@ -2,6 +2,7 @@ import static com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus.AVAILABLE; import static com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus.START_DATAHUB_STARTED; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -42,6 +43,7 @@ import com.sequenceiq.cloudbreak.workspace.model.Tenant; import com.sequenceiq.cloudbreak.workspace.model.Workspace; import com.sequenceiq.distrox.api.v1.distrox.model.DistroXV1Request; +import com.sequenceiq.distrox.api.v1.distrox.model.image.DistroXImageV1Request; import com.sequenceiq.distrox.v1.distrox.StackOperations; import com.sequenceiq.distrox.v1.distrox.converter.DistroXV1RequestToStackV4RequestConverter; import com.sequenceiq.distrox.v1.distrox.fedramp.FedRampModificationService; @@ -239,6 +241,7 @@ public void testIfDlIsNotExists() { DetailedEnvironmentResponse envResponse = new DetailedEnvironmentResponse(); envResponse.setEnvironmentStatus(AVAILABLE); envResponse.setCrn("crn"); + envResponse.setName(envName); DescribeFreeIpaResponse freeipa = new DescribeFreeIpaResponse(); freeipa.setAvailabilityStatus(AvailabilityStatus.AVAILABLE); freeipa.setStatus(com.sequenceiq.freeipa.api.v1.freeipa.stack.model.common.Status.AVAILABLE); @@ -246,7 +249,9 @@ public void testIfDlIsNotExists() { when(environmentClientService.getByName(envName)).thenReturn(envResponse); when(platformAwareSdxConnector.listSdxCrns(any(), any())).thenReturn(Set.of()); - assertThrows(BadRequestException.class, () -> underTest.post(request)); + assertThatThrownBy(() -> underTest.post(request)) + .isInstanceOf(BadRequestException.class) + .hasMessage("Data Lake stack cannot be found for environment CRN: %s (%s)", envName, envResponse.getCrn()); verify(platformAwareSdxConnector).listSdxCrns(any(), any()); verifyNoMoreInteractions(platformAwareSdxConnector); @@ -269,7 +274,9 @@ public void testIfDlIsNotRunning() { when(platformAwareSdxConnector.listSdxCrnsWithAvailability(any(), any(), any())) .thenReturn(Set.of(Pair.of(DATALAKE_CRN, StatusCheckResult.NOT_AVAILABLE))); - assertThrows(BadRequestException.class, () -> underTest.post(request)); + assertThatThrownBy(() -> underTest.post(request)) + .isInstanceOf(BadRequestException.class) + .hasMessage("Data Lake stacks of environment should be available."); verify(platformAwareSdxConnector).listSdxCrns(any(), any()); verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any()); @@ -333,6 +340,40 @@ public void testIfDlRollingUpgradeInProgress() { verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any()); } + @Test + void testIfImageIdAndOsBothSet() { + String envName = "someAwesomeEnvironment"; + DistroXV1Request request = new DistroXV1Request(); + request.setEnvironmentName(envName); + DistroXImageV1Request imageRequest = new DistroXImageV1Request(); + imageRequest.setId("id"); + imageRequest.setOs("os"); + request.setImage(imageRequest); + DetailedEnvironmentResponse envResponse = new DetailedEnvironmentResponse(); + envResponse.setEnvironmentStatus(AVAILABLE); + envResponse.setCrn("crn"); + DescribeFreeIpaResponse freeipa = new DescribeFreeIpaResponse(); + freeipa.setAvailabilityStatus(AvailabilityStatus.AVAILABLE); + freeipa.setStatus(com.sequenceiq.freeipa.api.v1.freeipa.stack.model.common.Status.AVAILABLE); + Workspace workspace = new Workspace(); + Tenant tenant = new Tenant(); + tenant.setName("test"); + workspace.setTenant(tenant); + when(workspaceService.getForCurrentUser()).thenReturn(workspace); + when(freeipaClientService.getByEnvironmentCrn("crn")).thenReturn(freeipa); + when(environmentClientService.getByName(envName)).thenReturn(envResponse); + when(platformAwareSdxConnector.listSdxCrns(any(), any())).thenReturn(Set.of(DATALAKE_CRN)); + when(platformAwareSdxConnector.listSdxCrnsWithAvailability(any(), any(), any())) + .thenReturn(Set.of(Pair.of(DATALAKE_CRN, StatusCheckResult.AVAILABLE))); + + assertThatThrownBy(() -> underTest.post(request)) + .isInstanceOf(BadRequestException.class) + .hasMessage("Image request can not have both image id and os parameters set."); + + verify(platformAwareSdxConnector).listSdxCrns(any(), any()); + verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any()); + } + private static VerificationMode calledOnce() { return times(1); } diff --git a/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/ModelDescriptions.java b/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/ModelDescriptions.java index 64386673f52..e777225e33a 100644 --- a/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/ModelDescriptions.java +++ b/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/ModelDescriptions.java @@ -92,6 +92,8 @@ public class ModelDescriptions { public static final String RUNTIME_VERSION = "Runtime version."; + public static final String OS = "Operating system."; + public static final String DEFAULT_RUNTIME_VERSION = "Default runtime version."; public static final String LOCK_COMPONENTS = "Option to lock components during the upgrade."; diff --git a/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/SdxClusterRequest.java b/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/SdxClusterRequest.java index 9f9b52015bb..a45f4120ea8 100644 --- a/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/SdxClusterRequest.java +++ b/datalake-api/src/main/java/com/sequenceiq/sdx/api/model/SdxClusterRequest.java @@ -16,6 +16,9 @@ public class SdxClusterRequest extends SdxClusterRequestBase { @ApiModelProperty(ModelDescriptions.RUNTIME_VERSION) private String runtime; + @ApiModelProperty(ModelDescriptions.OS) + private String os; + @ApiModelProperty(ModelDescriptions.RECIPES) private Set recipes; @@ -27,6 +30,14 @@ public void setRuntime(String runtime) { this.runtime = runtime; } + public String getOs() { + return os; + } + + public void setOs(String os) { + this.os = os; + } + public Set getRecipes() { return recipes; } @@ -39,6 +50,7 @@ public void setRecipes(Set recipes) { public String toString() { return "SdxClusterRequest{" + "runtime='" + runtime + '\'' + + ", os='" + os + '\'' + ", recipes=" + recipes + "} " + super.toString(); } diff --git a/datalake/src/main/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogService.java b/datalake/src/main/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogService.java index 068780bfe0a..89e6148878c 100644 --- a/datalake/src/main/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogService.java +++ b/datalake/src/main/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogService.java @@ -46,7 +46,7 @@ public String getResourceCrnByResourceName(String resourceName) { } public ImageV4Response getImageResponseFromImageRequest(ImageSettingsV4Request imageSettingsV4Request, ImageCatalogPlatform imageCatalogPlatform) { - if (imageSettingsV4Request != null) { + if (imageSettingsV4Request != null && !StringUtils.isBlank(imageSettingsV4Request.getId())) { List images = getImagesMatchingRequest(imageSettingsV4Request); if (images != null) { String providerName = imageCatalogPlatform.nameToLowerCase(); diff --git a/datalake/src/main/java/com/sequenceiq/datalake/service/sdx/SdxService.java b/datalake/src/main/java/com/sequenceiq/datalake/service/sdx/SdxService.java index 5a04e7cd490..def6dee9956 100644 --- a/datalake/src/main/java/com/sequenceiq/datalake/service/sdx/SdxService.java +++ b/datalake/src/main/java/com/sequenceiq/datalake/service/sdx/SdxService.java @@ -418,7 +418,13 @@ public Pair createSdx(final String userCrn, final St public Pair createSdx(final String userCrn, final String name, final SdxClusterRequest sdxClusterRequest, final StackV4Request internalStackV4Request) { - ImageSettingsV4Request imageSettingsV4Request = Optional.ofNullable(internalStackV4Request).map(StackV4Request::getImage).orElse(null); + ImageSettingsV4Request imageSettingsV4Request = null; + if (internalStackV4Request != null && internalStackV4Request.getImage() != null) { + imageSettingsV4Request = internalStackV4Request.getImage(); + } else if (StringUtils.isNotBlank(sdxClusterRequest.getOs())) { + imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setOs(sdxClusterRequest.getOs()); + } return createSdx(userCrn, name, sdxClusterRequest, internalStackV4Request, imageSettingsV4Request); } @@ -448,15 +454,17 @@ private Pair createSdx(final String userCrn, final S validateSdxRequest(name, sdxClusterRequest.getEnvironment(), accountId); validateJavaVersion(sdxClusterRequest.getJavaVersion(), accountId); DetailedEnvironmentResponse environment = validateAndGetEnvironment(sdxClusterRequest.getEnvironment()); + validateImageRequest(imageSettingsV4Request); ImageCatalogPlatform imageCatalogPlatform = platformStringTransformer .getPlatformStringForImageCatalog(environment.getCloudPlatform(), isGovCloudEnvironment(environment)); ImageV4Response imageV4Response = imageCatalogService.getImageResponseFromImageRequest(imageSettingsV4Request, imageCatalogPlatform); validateInternalSdxRequest(internalStackV4Request, sdxClusterRequest); validateRuntimeAndImage(sdxClusterRequest, environment, imageSettingsV4Request, imageV4Response); String runtimeVersion = getRuntime(sdxClusterRequest, internalStackV4Request, imageV4Response); - String os = Optional.ofNullable(imageV4Response).map(ImageV4Response::getOs) - .or(() -> Optional.ofNullable(internalStackV4Request).map(StackV4Request::getImage).map(ImageSettingsV4Request::getOs)) - .orElse(null); + String os = Optional.ofNullable(sdxClusterRequest.getOs()) + .or(() -> Optional.ofNullable(imageV4Response).map(ImageV4Response::getOs)) + .or(() -> Optional.ofNullable(internalStackV4Request).map(StackV4Request::getImage).map(ImageSettingsV4Request::getOs)) + .orElse(null); validateCcmV2Requirement(environment, runtimeVersion); @@ -1098,6 +1106,12 @@ private void validateSdxResizeRequest(SdxCluster sdxCluster, String accountId, S } } + private void validateImageRequest(ImageSettingsV4Request imageSettingsV4Request) { + if (imageSettingsV4Request != null && StringUtils.isNoneBlank(imageSettingsV4Request.getId(), imageSettingsV4Request.getOs())) { + throw new BadRequestException("Image request can not have both image id and os parameters set."); + } + } + @VisibleForTesting void validateRuntimeAndImage(SdxClusterRequest clusterRequest, DetailedEnvironmentResponse environment, ImageSettingsV4Request imageSettingsV4Request, ImageV4Response imageV4Response) { @@ -1110,7 +1124,7 @@ void validateRuntimeAndImage(SdxClusterRequest clusterRequest, DetailedEnvironme validationBuilder.error("SDX cluster request must not specify both runtime version and image at the same time because image " + "decides runtime version."); } - } else if (imageSettingsV4Request != null && StringUtils.isBlank(clusterRequest.getRuntime())) { + } else if (isImageSpecified(imageSettingsV4Request) && StringUtils.isBlank(clusterRequest.getRuntime())) { if (cloudPlatform.equals(MOCK)) { clusterRequest.setRuntime(MEDIUM_DUTY_REQUIRED_VERSION); } else { @@ -1125,6 +1139,10 @@ void validateRuntimeAndImage(SdxClusterRequest clusterRequest, DetailedEnvironme } } + private boolean isImageSpecified(ImageSettingsV4Request imageSettingsV4Request) { + return imageSettingsV4Request != null && !StringUtils.isBlank(imageSettingsV4Request.getId()); + } + private void validateInternalSdxRequest(StackV4Request stackv4Request, SdxClusterRequest sdxClusterRequest) { ValidationResultBuilder validationResultBuilder = ValidationResult.builder(); if (stackv4Request != null) { diff --git a/datalake/src/test/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogServiceTest.java b/datalake/src/test/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogServiceTest.java index b2ca3cbecff..c69c8672bd2 100644 --- a/datalake/src/test/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogServiceTest.java +++ b/datalake/src/test/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogServiceTest.java @@ -131,6 +131,15 @@ void testGetImageResponseFromImageRequestWithoutImageSettingsRequest() { assertNull(actual); } + @Test + void testGetImageResponseFromImageRequestWithOsImageSettingsRequest() { + ImageSettingsV4Request imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setOs("os"); + ImageV4Response actual = ThreadBasedUserCrnProvider.doAs(USER_CRN, + () -> victim.getImageResponseFromImageRequest(imageSettingsV4Request, IMAGE_CATALOG_PLATFORM_AWS)); + assertNull(actual); + } + @Test void testGetImageResponseFromImageRequestWithImageSettingsRequestNotResultingInImages() throws Exception { ImageSettingsV4Request imageSettingsV4Request = new ImageSettingsV4Request(); @@ -146,10 +155,12 @@ void testGetImageResponseFromImageRequestWithImageSettingsRequestNotResultingInI @Test void testGetImageResponseFromImageRequestWhenEndpointFails() throws Exception { + ImageSettingsV4Request imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setId(IMAGE_ID); doThrow(new Exception()).when(imageCatalogV4Endpoint).getImageByImageId(any(), any(), any()); ImageV4Response actual = ThreadBasedUserCrnProvider.doAs(USER_CRN, - () -> victim.getImageResponseFromImageRequest(new ImageSettingsV4Request(), IMAGE_CATALOG_PLATFORM_AWS)); + () -> victim.getImageResponseFromImageRequest(imageSettingsV4Request, IMAGE_CATALOG_PLATFORM_AWS)); assertNull(actual); verify(imageCatalogV4Endpoint).getImageByImageId(any(), any(), any()); } diff --git a/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceCreateSdxTest.java b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceCreateSdxTest.java new file mode 100644 index 00000000000..2c273a9789a --- /dev/null +++ b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceCreateSdxTest.java @@ -0,0 +1,1117 @@ +package com.sequenceiq.datalake.service.sdx; + +import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.AWS; +import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.AZURE; +import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.GCP; +import static com.sequenceiq.common.api.type.InstanceGroupType.CORE; +import static com.sequenceiq.common.api.type.InstanceGroupType.GATEWAY; +import static com.sequenceiq.datalake.service.sdx.SdxService.CCMV2_JUMPGATE_REQUIRED_VERSION; +import static com.sequenceiq.datalake.service.sdx.SdxService.CCMV2_REQUIRED_VERSION; +import static com.sequenceiq.datalake.service.sdx.SdxService.MEDIUM_DUTY_REQUIRED_VERSION; +import static com.sequenceiq.sdx.api.model.SdxClusterShape.CUSTOM; +import static com.sequenceiq.sdx.api.model.SdxClusterShape.ENTERPRISE; +import static com.sequenceiq.sdx.api.model.SdxClusterShape.LIGHT_DUTY; +import static com.sequenceiq.sdx.api.model.SdxClusterShape.MEDIUM_DUTY_HA; +import static com.sequenceiq.sdx.api.model.SdxClusterShape.MICRO_DUTY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Supplier; + +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.sequenceiq.authorization.service.OwnerAssignmentService; +import com.sequenceiq.cloudbreak.api.endpoint.v4.imagecatalog.responses.BaseStackDetailsV4Response; +import com.sequenceiq.cloudbreak.api.endpoint.v4.imagecatalog.responses.ImageV4Response; +import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.RecipeV4Endpoint; +import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.responses.RecipeViewV4Response; +import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.responses.RecipeViewV4Responses; +import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.StackV4Request; +import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request; +import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.image.ImageSettingsV4Request; +import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.instancegroup.InstanceGroupV4Request; +import com.sequenceiq.cloudbreak.auth.ThreadBasedUserCrnProvider; +import com.sequenceiq.cloudbreak.auth.altus.EntitlementService; +import com.sequenceiq.cloudbreak.auth.crn.CrnTestUtil; +import com.sequenceiq.cloudbreak.auth.crn.RegionAwareCrnGenerator; +import com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGenerator; +import com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGeneratorFactory; +import com.sequenceiq.cloudbreak.common.exception.BadRequestException; +import com.sequenceiq.cloudbreak.common.json.JsonUtil; +import com.sequenceiq.cloudbreak.common.mappable.CloudPlatform; +import com.sequenceiq.cloudbreak.common.service.Clock; +import com.sequenceiq.cloudbreak.common.service.PlatformStringTransformer; +import com.sequenceiq.cloudbreak.common.service.TransactionService; +import com.sequenceiq.cloudbreak.util.FileReaderUtils; +import com.sequenceiq.cloudbreak.vm.VirtualMachineConfiguration; +import com.sequenceiq.common.api.cloudstorage.old.S3CloudStorageV1Parameters; +import com.sequenceiq.common.api.type.Tunnel; +import com.sequenceiq.common.model.FileSystemType; +import com.sequenceiq.datalake.configuration.CDPConfigService; +import com.sequenceiq.datalake.configuration.PlatformConfig; +import com.sequenceiq.datalake.entity.DatalakeStatusEnum; +import com.sequenceiq.datalake.entity.SdxCluster; +import com.sequenceiq.datalake.flow.SdxReactorFlowManager; +import com.sequenceiq.datalake.repository.SdxClusterRepository; +import com.sequenceiq.datalake.service.EnvironmentClientService; +import com.sequenceiq.datalake.service.imagecatalog.ImageCatalogService; +import com.sequenceiq.datalake.service.sdx.status.SdxStatusService; +import com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse; +import com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus; +import com.sequenceiq.flow.api.model.FlowIdentifier; +import com.sequenceiq.flow.api.model.FlowType; +import com.sequenceiq.sdx.api.model.SdxAwsRequest; +import com.sequenceiq.sdx.api.model.SdxAwsSpotParameters; +import com.sequenceiq.sdx.api.model.SdxCloudStorageRequest; +import com.sequenceiq.sdx.api.model.SdxClusterRequest; +import com.sequenceiq.sdx.api.model.SdxClusterRequestBase; +import com.sequenceiq.sdx.api.model.SdxClusterResizeRequest; +import com.sequenceiq.sdx.api.model.SdxClusterShape; +import com.sequenceiq.sdx.api.model.SdxCustomClusterRequest; +import com.sequenceiq.sdx.api.model.SdxDatabaseRequest; +import com.sequenceiq.sdx.api.model.SdxInstanceGroupRequest; +import com.sequenceiq.sdx.api.model.SdxRecipe; + +@ExtendWith(MockitoExtension.class) +class SdxServiceCreateSdxTest { + + private static final Map TAGS = Collections.singletonMap("mytag", "tagecske"); + + private static final String USER_CRN = "crn:cdp:iam:us-west-1:hortonworks:user:perdos@hortonworks.com"; + + private static final String ENVIRONMENT_CRN = "crn:cdp:environments:us-west-1:default:environment:e438a2db-d650-4132-ae62-242c5ba2f784"; + + private static final String DATALAKE_CRN = "crn:cdp:datalake:us-west-1:default:datalake:e438a2db-d650-4132-ae62-242c5ba2f784"; + + private static final Long SDX_ID = 2L; + + private static final String SDX_CRN = "crn"; + + private static final String CLUSTER_NAME = "test-sdx-cluster"; + + private static final String OS = "centos7"; + + @Mock + private SdxExternalDatabaseConfigurer externalDatabaseConfigurer; + + @Mock + private SdxClusterRepository sdxClusterRepository; + + @Mock + private SdxReactorFlowManager sdxReactorFlowManager; + + @Mock + private EnvironmentClientService environmentClientService; + + @Mock + private Clock clock; + + @Mock + private CloudStorageManifester cloudStorageManifester; + + @Mock + private SdxStatusService sdxStatusService; + + @Mock + private RecipeV4Endpoint recipeV4Endpoint; + + @Mock + private CDPConfigService cdpConfigService; + + @Mock + private OwnerAssignmentService ownerAssignmentService; + + @Mock + private TransactionService transactionService; + + @Mock + private EntitlementService entitlementService; + + @Mock + private RegionAwareCrnGenerator regionAwareCrnGenerator; + + @Mock + private ImageCatalogService imageCatalogService; + + @Mock + private RegionAwareInternalCrnGeneratorFactory regionAwareInternalCrnGeneratorFactory; + + @Mock + private RegionAwareInternalCrnGenerator regionAwareInternalCrnGenerator; + + @Mock + private PlatformConfig platformConfig; + + @Mock + private VirtualMachineConfiguration virtualMachineConfiguration; + + @Mock + private PlatformStringTransformer platformStringTransformer; + + @InjectMocks + private SdxService underTest; + + static Object[][] razCloudPlatformDataProvider() { + return new Object[][]{ + // testCaseName cloudPlatform + {"CloudPlatform.AWS multiaz=true", AWS, true}, + {"CloudPlatform.AWS multiaz=false", AWS, false}, + {"CloudPlatform.AZURE multiaz=false", AZURE, false}, + {"CloudPlatform.GCP multiaz=false", GCP, false} + }; + } + + static Object[][] razCloudPlatformAndRuntimeDataProvider() { + return new Object[][]{ + // testCaseName cloudPlatform runtime + {"CloudPlatform.AWS", AWS, "7.2.2"}, + {"CloudPlatform.AZURE", AZURE, "7.2.2"}, + {"CloudPlatform.GCP", GCP, "7.2.17"} + }; + } + + static Object[][] razCloudPlatform710DataProvider() { + return new Object[][]{ + // testCaseName cloudPlatform expectedErrorMsg + {"CloudPlatform.AWS", AWS, + "Provisioning Ranger Raz on Amazon Web Services is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.2 and not 7.1.0"}, + {"CloudPlatform.AZURE", AZURE, + "Provisioning Ranger Raz on Microsoft Azure is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.2 and not 7.1.0"}, + {"CloudPlatform.GCP", GCP, + "Provisioning Ranger Raz on GCP is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.17 and not 7.1.0"} + }; + } + + static Object[][] razCloudPlatform720DataProvider() { + return new Object[][]{ + // testCaseName cloudPlatform expectedErrorMsg + {"CloudPlatform.AWS", AWS, + "Provisioning Ranger Raz on Amazon Web Services is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.2 and not 7.2.0"}, + {"CloudPlatform.AZURE", AZURE, + "Provisioning Ranger Raz on Microsoft Azure is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.2 and not 7.2.0"}, + {"CloudPlatform.GCP", GCP, + "Provisioning Ranger Raz on GCP is only valid for Cloudera Runtime version " + + "greater than or equal to 7.2.17 and not 7.2.0"} + }; + } + + static Object[][] ccmV2Scenarios() { + return new Object[][]{ + // runtime compatible + {null, true}, + {"7.2.0", false}, + {"7.2.1", true}, + {"7.2.5", true}, + {"7.2.6", true}, + {"7.2.7", true}, + }; + } + + static Object[][] ccmV2JumpgateScenarios() { + return new Object[][]{ + // runtime compatible + {null, true}, + {"7.2.0", false}, + {"7.2.1", false}, + {"7.2.5", false}, + {"7.2.6", true}, + {"7.2.7", true}, + }; + } + + static Object[][] deleteInProgressParamProvider() { + return new Object[][]{ + {EnvironmentStatus.DELETE_INITIATED}, + {EnvironmentStatus.NETWORK_DELETE_IN_PROGRESS}, + {EnvironmentStatus.RDBMS_DELETE_IN_PROGRESS}, + {EnvironmentStatus.FREEIPA_DELETE_IN_PROGRESS}, + {EnvironmentStatus.CLUSTER_DEFINITION_CLEANUP_PROGRESS}, + {EnvironmentStatus.UMS_RESOURCE_DELETE_IN_PROGRESS}, + {EnvironmentStatus.IDBROKER_MAPPINGS_DELETE_IN_PROGRESS}, + {EnvironmentStatus.S3GUARD_TABLE_DELETE_IN_PROGRESS}, + {EnvironmentStatus.DATAHUB_CLUSTERS_DELETE_IN_PROGRESS}, + {EnvironmentStatus.DATALAKE_CLUSTERS_DELETE_IN_PROGRESS}, + {EnvironmentStatus.PUBLICKEY_DELETE_IN_PROGRESS} + }; + } + + static Object[][] failedParamProvider() { + return new Object[][]{ + {EnvironmentStatus.CREATE_FAILED}, + {EnvironmentStatus.DELETE_FAILED}, + {EnvironmentStatus.UPDATE_FAILED}, + {EnvironmentStatus.FREEIPA_DELETED_ON_PROVIDER_SIDE} + }; + } + + @BeforeEach + void initMocks() { + lenient().when(platformConfig.getRazSupportedPlatforms()) + .thenReturn(List.of(AWS, AZURE, GCP)); + lenient().when(entitlementService.isRazForGcpEnabled(anyString())) + .thenReturn(true); + } + + @Test + void testCreateSdxClusterWhenClusterWithSpecifiedNameAlreadyExistShouldThrowClusterAlreadyExistBadRequestException() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); + SdxCluster existing = new SdxCluster(); + existing.setEnvName("envir"); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse( + anyString(), anyString())).thenReturn(Collections.singletonList(existing)); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("SDX cluster exists for environment name: envir", badRequestException.getMessage()); + } + + @Test + void testCreateSdxClusterWithoutCloudStorageShouldThrownBadRequestException() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Cloud storage parameter is required.", badRequestException.getMessage()); + } + + @Test + void testCreateSdxClusterWithoutCloudStorageShouldNotThrownBadRequestExceptionInCaseOfInternal() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", CUSTOM); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + StackV4Request stackV4Request = new StackV4Request(); + ClusterV4Request clusterV4Request = new ClusterV4Request(); + stackV4Request.setCluster(clusterV4Request); + ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); + } + + @Test + void testNullJavaVersionShouldNotOverrideTheVersionInTheInternalStackRequest() throws IOException, TransactionService.TransactionExecutionException { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", CUSTOM); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AWS, null); + StackV4Request stackV4Request = new StackV4Request(); + stackV4Request.setJavaVersion(8); + ClusterV4Request clusterV4Request = new ClusterV4Request(); + stackV4Request.setCluster(clusterV4Request); + + ArgumentCaptor sdxClusterArgumentCaptor = ArgumentCaptor.forClass(SdxCluster.class); + when(sdxClusterRepository.save(sdxClusterArgumentCaptor.capture())).thenReturn(mock(SdxCluster.class)); + + ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); + + StackV4Request savedStackV4Request = JsonUtil.readValue(sdxClusterArgumentCaptor.getValue().getStackRequest(), StackV4Request.class); + assertEquals(8, savedStackV4Request.getJavaVersion()); + } + + @Test + void testCreateNOTInternalSdxClusterFromLightDutyTemplateShouldTriggerSdxCreationFlow() + throws IOException, TransactionService.TransactionExecutionException { + CrnTestUtil.mockCrnGenerator(regionAwareCrnGenerator); + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, AZURE, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + SdxCluster capturedSdx = captor.getValue(); + assertEquals("tagecske", capturedSdx.getTags().getValue("mytag")); + assertEquals(CLUSTER_NAME, capturedSdx.getClusterName()); + assertEquals(LIGHT_DUTY, capturedSdx.getClusterShape()); + assertEquals("envir", capturedSdx.getEnvName()); + assertEquals("hortonworks", capturedSdx.getAccountId()); + verify(sdxStatusService, times(1)).setStatusForDatalakeAndNotify(DatalakeStatusEnum.REQUESTED, "Datalake requested", createdSdxCluster); + + assertEquals(1L, capturedSdx.getCreated()); + assertFalse(capturedSdx.isCreateDatabase()); + assertTrue(createdSdxCluster.getCrn().matches("crn:cdp:datalake:us-west-1:hortonworks:datalake:.*")); + StackV4Request stackV4Request = JsonUtil.readValue(capturedSdx.getStackRequest(), StackV4Request.class); + + assertEquals(2L, stackV4Request.getInstanceGroups().size()); + + InstanceGroupV4Request core = getGroup(stackV4Request, CORE); + assertEquals(1L, core.getSecurityGroup().getSecurityRules().size()); + assertEquals("0.0.0.0/0", core.getSecurityGroup().getSecurityRules().get(0).getSubnet()); + assertEquals("22", core.getSecurityGroup().getSecurityRules().get(0).getPorts().get(0)); + + InstanceGroupV4Request gateway = getGroup(stackV4Request, GATEWAY); + assertEquals(2L, gateway.getSecurityGroup().getSecurityRules().size()); + assertEquals("0.0.0.0/0", gateway.getSecurityGroup().getSecurityRules().get(0).getSubnet()); + assertEquals("443", gateway.getSecurityGroup().getSecurityRules().get(0).getPorts().get(0)); + assertEquals("0.0.0.0/0", gateway.getSecurityGroup().getSecurityRules().get(1).getSubnet()); + assertEquals("22", gateway.getSecurityGroup().getSecurityRules().get(1).getPorts().get(0)); + + verify(sdxReactorFlowManager).triggerSdxCreation(createdSdxCluster); + } + + @Test + void testCreateNOTInternalSdxClusterFromLightDutyTemplateWhenLocationSpecifiedWithSlashShouldCreateAndSettedUpBaseLocationWithOUTSlash() + throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals("s3a://some/dir", createdSdxCluster.getCloudStorageBaseLocation()); + } + + @Test + void testCreateSdxClusterWithSpotStackRequestContainsRequiredAttributes() throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); + setSpot(sdxClusterRequest); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + // AWS 7.1.0 light duty contains exactly 2 instance groups + assertThat(createdSdxCluster.getStackRequest()).containsSubsequence( + "{\"aws\":{\"spot\":{\"percentage\":100,\"maxPrice\":0.9}}", + "{\"aws\":{\"spot\":{\"percentage\":100,\"maxPrice\":0.9}}"); + } + + @Test + void testCreateSdxClusterWithJavaVersionStackRequestContainsRequiredAttributes() throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + when(virtualMachineConfiguration.getSupportedJavaVersions()).thenReturn(Set.of(11)); + + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); + sdxClusterRequest.setJavaVersion(11); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + + assertThat(createdSdxCluster.getStackRequest()).containsSubsequence("\"javaVersion\":11"); + } + + @Test + void testCreateNOTInternalSdxClusterFromLightDutyTemplateWhenBaseLocationSpecifiedShouldCreateStackRequestWithSettedUpBaseLocation() + throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + //doNothing().when(cloudStorageLocationValidator.validate("s3a://some/dir", )); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals("s3a://some/dir", createdSdxCluster.getCloudStorageBaseLocation()); + } + + @MethodSource("startParamProvider") + void testCreateButEnvInStoppedStatus(EnvironmentStatus environmentStatus, String exceptionMessage) { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); + + DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); + detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); + detailedEnvironmentResponse.setCloudPlatform(AWS.name()); + detailedEnvironmentResponse.setCrn(getCrn()); + detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); + when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); + + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), + "BadRequestException should thrown"); + assertEquals(exceptionMessage, badRequestException.getMessage()); + } + + @ParameterizedTest + @MethodSource("deleteInProgressParamProvider") + void testCreateButEnvInDeleteInProgressPhase(EnvironmentStatus environmentStatus) { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); + DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); + detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); + detailedEnvironmentResponse.setCloudPlatform(AWS.name()); + detailedEnvironmentResponse.setCrn(getCrn()); + detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); + when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); + + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), + "BadRequestException should thrown"); + assertEquals("The environment is in delete in progress phase. Please create a new environment first!", badRequestException.getMessage()); + } + + @ParameterizedTest + @MethodSource("failedParamProvider") + void testCreateButEnvInFailedPhase(EnvironmentStatus environmentStatus) { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); + DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); + detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); + detailedEnvironmentResponse.setCloudPlatform(AWS.name()); + detailedEnvironmentResponse.setCrn(getCrn()); + detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); + when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); + + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), + "BadRequestException should thrown"); + assertEquals("The environment is in failed phase. Please fix the environment or create a new one first!", badRequestException.getMessage()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("razCloudPlatformDataProvider") + void testSdxCreateRazNotRequestedAndMultiAzRequested(String testCaseName, CloudPlatform cloudPlatform, boolean multiAz) + throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.1.0", LIGHT_DUTY); + withCloudStorage(sdxClusterRequest); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); + sdxClusterRequest.setEnableRangerRaz(false); + sdxClusterRequest.setEnableMultiAz(multiAz); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + SdxCluster capturedSdx = captor.getValue(); + assertFalse(capturedSdx.isRangerRazEnabled()); + assertEquals(multiAz, capturedSdx.isEnableMultiAz()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("razCloudPlatformAndRuntimeDataProvider") + void testSdxCreateRazEnabled(String testCaseName, CloudPlatform cloudPlatform, String runtime) + throws IOException, TransactionService.TransactionExecutionException { + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); + sdxClusterRequest.setEnableRangerRaz(true); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + SdxCluster capturedSdx = captor.getValue(); + assertTrue(capturedSdx.isRangerRazEnabled()); + } + + @Test + void testSdxCreateRazEnabledWithRazNotEnabledCloud() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.2", LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.YARN, null); + sdxClusterRequest.setEnableRangerRaz(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning Ranger Raz is only valid for Amazon Web Services, Microsoft Azure, GCP", badRequestException.getMessage()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("razCloudPlatform710DataProvider") + void testSdxCreateRazEnabled710Runtime(String testCaseName, CloudPlatform cloudPlatform, String expectedErrorMsg) { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.1.0", LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); + sdxClusterRequest.setEnableRangerRaz(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals(expectedErrorMsg, badRequestException.getMessage()); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("razCloudPlatform720DataProvider") + void testSdxCreateRazEnabled720Runtime(String testCaseName, CloudPlatform cloudPlatform, String expectedErrorMsg) { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.0", LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); + sdxClusterRequest.setEnableRangerRaz(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals(expectedErrorMsg, badRequestException.getMessage()); + } + + @Test + void testSdxCreateRazEnabledForGcpEntitilementNotEnabled() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.17", LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + when(entitlementService.isRazForGcpEnabled(anyString())).thenReturn(false); + mockEnvironmentCall(sdxClusterRequest, GCP, null); + + sdxClusterRequest.setEnableRangerRaz(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning Ranger Raz on GCP is not enabled for this account", badRequestException.getMessage()); + } + + @Test + void testSdxCreateWithBothImageIdAndOsSet() throws IOException { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.17", LIGHT_DUTY); + withCloudStorage(sdxClusterRequest); + StackV4Request stackV4Request = new StackV4Request(); + ImageSettingsV4Request imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setId("id"); + imageSettingsV4Request.setOs("os"); + stackV4Request.setImage(imageSettingsV4Request); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request))); + assertEquals("Image request can not have both image id and os parameters set.", badRequestException.getMessage()); + } + + @Test + void testSdxCreateMediumDutySdx() throws IOException, TransactionService.TransactionExecutionException { + final String runtime = "7.2.7"; + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/medium_duty_ha.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MEDIUM_DUTY_HA); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + SdxCluster capturedSdx = captor.getValue(); + assertEquals(MEDIUM_DUTY_HA, capturedSdx.getClusterShape()); + } + + @Test + void testSdxCreateMediumDutySdx710Runtime() { + final String invalidRuntime = "7.1.0"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(invalidRuntime, MEDIUM_DUTY_HA); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning a Medium Duty SDX shape is only valid for CM version greater than or equal to " + + MEDIUM_DUTY_REQUIRED_VERSION + " and not " + invalidRuntime, badRequestException.getMessage()); + } + + @Test + void testSdxCreateMediumDutySdx720Runtime() { + final String invalidRuntime = "7.2.0"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(invalidRuntime, MEDIUM_DUTY_HA); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning a Medium Duty SDX shape is only valid for CM version greater than or equal to " + + MEDIUM_DUTY_REQUIRED_VERSION + " and not " + invalidRuntime, badRequestException.getMessage()); + } + + @Test + void testCreateSdxClusterWithCustomRequestContainsImageInfo() throws Exception { + ImageV4Response imageResponse = getImageResponse(); + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.7/aws/light_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + SdxCustomClusterRequest sdxClusterRequest = createSdxCustomClusterRequest(LIGHT_DUTY, "cdp-default", "imageId_1"); + setSpot(sdxClusterRequest); + withCloudStorage(sdxClusterRequest); + when(imageCatalogService.getImageResponseFromImageRequest(eq(sdxClusterRequest.getImageSettingsV4Request()), any())).thenReturn(imageResponse); + + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, + () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest)); + + SdxCluster createdSdxCluster = result.getLeft(); + StackV4Request stackV4Request = JsonUtil.readValue(createdSdxCluster.getStackRequest(), StackV4Request.class); + assertNotNull(stackV4Request.getImage()); + assertEquals("cdp-default", stackV4Request.getImage().getCatalog()); + assertEquals("imageId_1", stackV4Request.getImage().getId()); + verify(externalDatabaseConfigurer).configure(any(), eq(OS), any(), any(), any()); + } + + @Test + void testCreateInternalSdxClusterWithCustomInstanceGroupShouldFail() { + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + StackV4Request stackV4Request = new StackV4Request(); + ClusterV4Request clusterV4Request = new ClusterV4Request(); + stackV4Request.setCluster(clusterV4Request); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.12", CUSTOM); + withCustomInstanceGroups(sdxClusterRequest); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); + assertEquals("Custom instance group is not accepted on SDX Internal API.", badRequestException.getMessage()); + } + + @Test + void testCreateSdxClusterWithCustomInstanceGroup() throws Exception { + final String runtime = "7.2.12"; + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String microDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/micro_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(microDutyJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + withRecipe(sdxClusterRequest); + withCustomInstanceGroups(sdxClusterRequest); + RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); + RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); + recipeViewV4Response.setName("post-service-deployment"); + recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); + when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); + when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); + + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); + SdxCluster capturedSdx = captor.getValue(); + assertEquals(MICRO_DUTY, capturedSdx.getClusterShape()); + StackV4Request stackRequest = JsonUtil.readValue(capturedSdx.getStackRequest(), StackV4Request.class); + Optional masterGroup = stackRequest.getInstanceGroups().stream() + .filter(instanceGroup -> "master".equals(instanceGroup.getName())) + .findAny(); + assertTrue(masterGroup.isPresent()); + assertEquals("verylarge", masterGroup.get().getTemplate().getInstanceType()); + Optional idbrokerGroup = stackRequest.getInstanceGroups().stream() + .filter(instanceGroup -> "idbroker".equals(instanceGroup.getName())) + .findAny(); + assertTrue(idbrokerGroup.isPresent()); + assertEquals("notverylarge", idbrokerGroup.get().getTemplate().getInstanceType()); + } + + @ParameterizedTest(name = "Runtime {0} is compatible with CCMv2 = {1}") + @MethodSource("ccmV2Scenarios") + void testCcmV2VersionChecker(String runtime, boolean compatible) throws IOException { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.10/aws/light_duty.json"); + lenient().when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.MOCK, Tunnel.CCMV2); + if (!compatible) { + assertThatThrownBy(() -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))) + .isInstanceOf(BadRequestException.class) + .hasMessage(String.format("Runtime version %s does not support Cluster Connectivity Manager. " + + "Please try creating a datalake with runtime version at least %s.", runtime, CCMV2_REQUIRED_VERSION)); + } else { + ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + } + } + + @ParameterizedTest(name = "Runtime {0} is compatible with CCMv2JumpGate = {1}") + @MethodSource("ccmV2JumpgateScenarios") + void testCcmV2JumpgateVersionChecker(String runtime, boolean compatible) throws IOException { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.10/aws/light_duty.json"); + lenient().when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.MOCK, Tunnel.CCMV2_JUMPGATE); + if (!compatible) { + assertThatThrownBy(() -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))) + .isInstanceOf(BadRequestException.class) + .hasMessage(String.format("Runtime version %s does not support Cluster Connectivity Manager. " + + "Please try creating a datalake with runtime version at least %s.", runtime, CCMV2_JUMPGATE_REQUIRED_VERSION)); + } else { + ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + } + } + + @Test + void testCreateMicroDuty() throws IOException, TransactionService.TransactionExecutionException { + final String runtime = "7.2.12"; + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String microDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/micro_duty.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(microDutyJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + withRecipe(sdxClusterRequest); + RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); + RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); + recipeViewV4Response.setName("post-service-deployment"); + recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); + when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); + when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); + when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, AWS, null); + when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); + SdxCluster capturedSdx = captor.getValue(); + assertEquals(MICRO_DUTY, capturedSdx.getClusterShape()); + } + + @Test + void testCreateMicroDutyWrongVersion() { + final String runtime = "7.2.11"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AZURE, null); + when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning a Micro Duty SDX shape is only valid for CM version greater than or equal to 7.2.12 and not 7.2.11", + badRequestException.getMessage()); + } + + @Test + void testCreateMicroDutyNoEntitlement() { + final String runtime = "7.2.12"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AZURE, null); + when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(false); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals(String.format("Provisioning a micro duty data lake cluster is not enabled for %s. ", AZURE.name()) + + "Contact Cloudera support to enable CDP_MICRO_DUTY_SDX entitlement for the account.", badRequestException.getMessage()); + } + + @Test + void testCreateEnterpriseDatalake() throws IOException, TransactionService.TransactionExecutionException { + final String runtime = "7.2.17"; + when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); + String enterpriseJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/enterprise.json"); + when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(enterpriseJson, StackV4Request.class)); + when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + withCloudStorage(sdxClusterRequest); + withRecipe(sdxClusterRequest); + RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); + RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); + recipeViewV4Response.setName("post-service-deployment"); + recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); + when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); + when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); + when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); + long id = 10L; + when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { + SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); + sdxWithId.setId(id); + return sdxWithId; + }); + when(clock.getCurrentTimeMillis()).thenReturn(1L); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AWS, null); + when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(false); + Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); + SdxCluster createdSdxCluster = result.getLeft(); + assertEquals(id, createdSdxCluster.getId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); + verify(sdxClusterRepository, times(1)).save(captor.capture()); + verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); + SdxCluster capturedSdx = captor.getValue(); + assertEquals(ENTERPRISE, capturedSdx.getClusterShape()); + } + + @Test + void testCreateEnterpriseDatalakeWrongVersion() { + final String runtime = "7.2.11"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AZURE, null); + when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(false); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning an Enterprise SDX shape is only valid for CM version greater than or equal to 7.2.17 and not 7.2.11", + badRequestException.getMessage()); + } + + @Test + void testCreateEnterpriseDatalakeWithDisabledEntitlement() { + final String runtime = "7.2.17"; + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); + mockEnvironmentCall(sdxClusterRequest, AZURE, null); + when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(true); + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Provisioning an enterprise data lake cluster is disabled. " + + "Contact Cloudera support to enable this scale for the account.", badRequestException.getMessage()); + } + + @Test + void testCreateSdxClusterFialsInCaseOfForcedJavaVersionIsNotSupportedByTheVirtualMachineConfiguration() { + SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); + sdxClusterRequest.setJavaVersion(11); + + when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse( + anyString(), anyString())).thenReturn(Collections.emptyList()); + when(virtualMachineConfiguration.getSupportedJavaVersions()).thenReturn(Collections.emptySet()); + + BadRequestException badRequestException = assertThrows(BadRequestException.class, + () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> + underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); + assertEquals("Java version 11 is not supported.", badRequestException.getMessage()); + } + + private void setSpot(SdxClusterRequestBase sdxClusterRequest) { + SdxAwsRequest aws = new SdxAwsRequest(); + SdxAwsSpotParameters spot = new SdxAwsSpotParameters(); + spot.setPercentage(100); + spot.setMaxPrice(0.9); + aws.setSpot(spot); + sdxClusterRequest.setAws(aws); + } + + private ImageV4Response getImageResponse() { + Map> imageSetsByProvider = new HashMap<>(); + imageSetsByProvider.put("aws", null); + BaseStackDetailsV4Response stackDetails = new BaseStackDetailsV4Response(); + stackDetails.setVersion("7.2.7"); + + ImageV4Response imageV4Response = new ImageV4Response(); + imageV4Response.setOs(OS); + imageV4Response.setImageSetsByProvider(imageSetsByProvider); + imageV4Response.setStackDetails(stackDetails); + return imageV4Response; + } + + private void mockEnvironmentCall(SdxClusterResizeRequest sdxClusterResizeRequest, CloudPlatform cloudPlatform) { + DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); + detailedEnvironmentResponse.setName(sdxClusterResizeRequest.getEnvironment()); + detailedEnvironmentResponse.setCloudPlatform(cloudPlatform.name()); + detailedEnvironmentResponse.setEnvironmentStatus(EnvironmentStatus.AVAILABLE); + detailedEnvironmentResponse.setCrn(getCrn()); + detailedEnvironmentResponse.setCreator(detailedEnvironmentResponse.getCrn()); + when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); + } + + private void mockEnvironmentCall(SdxClusterRequestBase sdxClusterRequest, CloudPlatform cloudPlatform, Tunnel tunnel) { + DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); + detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); + detailedEnvironmentResponse.setCloudPlatform(cloudPlatform.name()); + detailedEnvironmentResponse.setEnvironmentStatus(EnvironmentStatus.AVAILABLE); + detailedEnvironmentResponse.setCrn(getCrn()); + detailedEnvironmentResponse.setCreator(detailedEnvironmentResponse.getCrn()); + detailedEnvironmentResponse.setTunnel(tunnel); + when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); + } + + private String getCrn() { + return CrnTestUtil.getEnvironmentCrnBuilder() + .setResource(UUID.randomUUID().toString()) + .setAccountId(UUID.randomUUID().toString()) + .build().toString(); + } + + private SdxCluster getSdxCluster() { + SdxCluster sdxCluster = new SdxCluster(); + sdxCluster.setId(SDX_ID); + sdxCluster.setCrn(SDX_CRN); + sdxCluster.setEnvCrn(ENVIRONMENT_CRN); + sdxCluster.setEnvName("envir"); + sdxCluster.setClusterName("sdx-cluster-name"); + return sdxCluster; + } + + private InstanceGroupV4Request getGroup(StackV4Request stack, com.sequenceiq.common.api.type.InstanceGroupType type) { + for (InstanceGroupV4Request instanceGroup : stack.getInstanceGroups()) { + if (instanceGroup.getType().equals(type)) { + return instanceGroup; + } + } + return null; + } + + private SdxClusterRequest createSdxClusterRequest(String runtime, SdxClusterShape shape) { + SdxClusterRequest sdxClusterRequest = new SdxClusterRequest(); + sdxClusterRequest.setRuntime(runtime); + sdxClusterRequest.setClusterShape(shape); + sdxClusterRequest.addTags(TAGS); + sdxClusterRequest.setEnvironment("envir"); + sdxClusterRequest.setExternalDatabase(new SdxDatabaseRequest()); + return sdxClusterRequest; + } + + private void withRecipe(SdxClusterRequest sdxClusterRequest) { + SdxRecipe recipe = new SdxRecipe(); + recipe.setHostGroup("master"); + recipe.setName("post-service-deployment"); + sdxClusterRequest.setRecipes(Set.of(recipe)); + } + + private void withCloudStorage(SdxClusterRequestBase sdxClusterRequest) { + SdxCloudStorageRequest cloudStorage = new SdxCloudStorageRequest(); + cloudStorage.setFileSystemType(FileSystemType.S3); + cloudStorage.setBaseLocation("s3a://some/dir/"); + cloudStorage.setS3(new S3CloudStorageV1Parameters()); + sdxClusterRequest.setCloudStorage(cloudStorage); + } + + private void withCustomInstanceGroups(SdxClusterRequestBase sdxClusterRequest) { + sdxClusterRequest.setCustomInstanceGroups(List.of(withInstanceGroup("master", "verylarge"), + withInstanceGroup("idbroker", "notverylarge"))); + } + + private SdxInstanceGroupRequest withInstanceGroup(String name, String instanceType) { + SdxInstanceGroupRequest masterInstanceGroup = new SdxInstanceGroupRequest(); + masterInstanceGroup.setName(name); + masterInstanceGroup.setInstanceType(instanceType); + return masterInstanceGroup; + } + + private SdxCustomClusterRequest createSdxCustomClusterRequest(SdxClusterShape shape, String catalog, String imageId) { + ImageSettingsV4Request imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setCatalog(catalog); + imageSettingsV4Request.setId(imageId); + + SdxCustomClusterRequest sdxClusterRequest = new SdxCustomClusterRequest(); + sdxClusterRequest.setClusterShape(shape); + sdxClusterRequest.addTags(TAGS); + sdxClusterRequest.setEnvironment("envir"); + sdxClusterRequest.setImageSettingsV4Request(imageSettingsV4Request); + sdxClusterRequest.setExternalDatabase(new SdxDatabaseRequest()); + return sdxClusterRequest; + } + +} diff --git a/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceTest.java b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceTest.java index e19bb2d1478..0dd1b189f86 100644 --- a/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceTest.java +++ b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceTest.java @@ -3,27 +3,15 @@ import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.AWS; import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.AZURE; import static com.sequenceiq.cloudbreak.common.mappable.CloudPlatform.GCP; -import static com.sequenceiq.common.api.type.InstanceGroupType.CORE; -import static com.sequenceiq.common.api.type.InstanceGroupType.GATEWAY; -import static com.sequenceiq.datalake.service.sdx.SdxService.CCMV2_JUMPGATE_REQUIRED_VERSION; -import static com.sequenceiq.datalake.service.sdx.SdxService.CCMV2_REQUIRED_VERSION; import static com.sequenceiq.datalake.service.sdx.SdxService.MEDIUM_DUTY_REQUIRED_VERSION; import static com.sequenceiq.datalake.service.sdx.SdxService.WORKSPACE_ID_DEFAULT; -import static com.sequenceiq.sdx.api.model.SdxClusterShape.CUSTOM; -import static com.sequenceiq.sdx.api.model.SdxClusterShape.ENTERPRISE; import static com.sequenceiq.sdx.api.model.SdxClusterShape.LIGHT_DUTY; import static com.sequenceiq.sdx.api.model.SdxClusterShape.MEDIUM_DUTY_HA; -import static com.sequenceiq.sdx.api.model.SdxClusterShape.MICRO_DUTY; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.anyString; @@ -37,7 +25,6 @@ import static org.mockito.Mockito.when; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -60,25 +47,17 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; -import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.mockito.junit.jupiter.MockitoExtension; -import com.sequenceiq.authorization.service.OwnerAssignmentService; import com.sequenceiq.cloudbreak.api.endpoint.v4.common.Status; import com.sequenceiq.cloudbreak.api.endpoint.v4.dto.NameOrCrn; -import com.sequenceiq.cloudbreak.api.endpoint.v4.imagecatalog.ImageCatalogV4Endpoint; import com.sequenceiq.cloudbreak.api.endpoint.v4.imagecatalog.responses.BaseStackDetailsV4Response; import com.sequenceiq.cloudbreak.api.endpoint.v4.imagecatalog.responses.ImageV4Response; -import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.RecipeV4Endpoint; -import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.responses.RecipeViewV4Response; -import com.sequenceiq.cloudbreak.api.endpoint.v4.recipes.responses.RecipeViewV4Responses; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.StackV4Endpoint; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.RotateSaltPasswordRequest; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.StackV4Request; -import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.cluster.ClusterV4Request; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.image.ImageSettingsV4Request; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.request.instancegroup.InstanceGroupV4Request; import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.response.RangerRazEnabledV4Response; @@ -93,21 +72,16 @@ import com.sequenceiq.cloudbreak.auth.crn.RegionAwareCrnGenerator; import com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGenerator; import com.sequenceiq.cloudbreak.auth.crn.RegionAwareInternalCrnGeneratorFactory; -import com.sequenceiq.cloudbreak.client.CloudbreakInternalCrnClient; -import com.sequenceiq.cloudbreak.client.CloudbreakServiceCrnEndpoints; -import com.sequenceiq.cloudbreak.cloud.model.catalog.Image; import com.sequenceiq.cloudbreak.common.exception.BadRequestException; import com.sequenceiq.cloudbreak.common.exception.NotFoundException; import com.sequenceiq.cloudbreak.common.json.JsonUtil; import com.sequenceiq.cloudbreak.common.mappable.CloudPlatform; import com.sequenceiq.cloudbreak.common.service.Clock; -import com.sequenceiq.cloudbreak.common.service.PlatformStringTransformer; import com.sequenceiq.cloudbreak.common.service.TransactionService; import com.sequenceiq.cloudbreak.common.service.TransactionService.TransactionExecutionException; import com.sequenceiq.cloudbreak.datalakedr.DatalakeDrSkipOptions; import com.sequenceiq.cloudbreak.util.FileReaderUtils; import com.sequenceiq.cloudbreak.validation.ValidationResult; -import com.sequenceiq.cloudbreak.vm.VirtualMachineConfiguration; import com.sequenceiq.common.api.cloudstorage.old.S3CloudStorageV1Parameters; import com.sequenceiq.common.api.type.Tunnel; import com.sequenceiq.common.model.FileSystemType; @@ -116,17 +90,14 @@ import com.sequenceiq.datalake.configuration.PlatformConfig; import com.sequenceiq.datalake.entity.DatalakeStatusEnum; import com.sequenceiq.datalake.entity.SdxCluster; -import com.sequenceiq.datalake.entity.SdxDatabase; import com.sequenceiq.datalake.entity.SdxStatusEntity; import com.sequenceiq.datalake.flow.SdxReactorFlowManager; import com.sequenceiq.datalake.repository.SdxClusterRepository; import com.sequenceiq.datalake.repository.SdxDatabaseRepository; import com.sequenceiq.datalake.service.EnvironmentClientService; -import com.sequenceiq.datalake.service.imagecatalog.ImageCatalogService; import com.sequenceiq.datalake.service.sdx.dr.SdxBackupRestoreService; import com.sequenceiq.datalake.service.sdx.flowcheck.CloudbreakFlowService; import com.sequenceiq.datalake.service.sdx.status.SdxStatusService; -import com.sequenceiq.datalake.service.validation.cloudstorage.CloudStorageLocationValidator; import com.sequenceiq.distrox.api.v1.distrox.endpoint.DistroXV1Endpoint; import com.sequenceiq.environment.api.v1.environment.model.response.DetailedEnvironmentResponse; import com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus; @@ -164,9 +135,6 @@ class SdxServiceTest { private static final String OS = "centos7"; - @Mock - private SdxExternalDatabaseConfigurer externalDatabaseConfigurer; - @Mock private SdxClusterRepository sdxClusterRepository; @@ -176,9 +144,6 @@ class SdxServiceTest { @Mock private EnvironmentClientService environmentClientService; - @Mock - private StackRequestManifester stackRequestManifester; - @Mock private Clock clock; @@ -191,51 +156,24 @@ class SdxServiceTest { @Mock private DistroXV1Endpoint distroXV1Endpoint; - @Mock - private SdxNotificationService notificationService; - @Mock private StackV4Endpoint stackV4Endpoint; - @Mock - private RecipeV4Endpoint recipeV4Endpoint; - - @Mock - private ImageCatalogV4Endpoint imageCatalogV4Endpoint; - - @Mock - private CloudbreakInternalCrnClient cloudbreakInternalCrnClient; - - @Mock - private CloudbreakServiceCrnEndpoints cloudbreakServiceCrnEndpoints; - @Mock private DistroxService distroxService; - @Mock - private CloudStorageLocationValidator cloudStorageLocationValidator; - @Mock private CDPConfigService cdpConfigService; - @Mock - private OwnerAssignmentService ownerAssignmentService; - @Mock private TransactionService transactionService; @Mock private EntitlementService entitlementService; - @Mock - private Image image; - @Mock private RegionAwareCrnGenerator regionAwareCrnGenerator; - @Mock - private ImageCatalogService imageCatalogService; - @Mock private RegionAwareInternalCrnGeneratorFactory regionAwareInternalCrnGeneratorFactory; @@ -248,12 +186,6 @@ class SdxServiceTest { @Mock private PlatformConfig platformConfig; - @Mock - private VirtualMachineConfiguration virtualMachineConfiguration; - - @Mock - private PlatformStringTransformer platformStringTransformer; - @Mock private ImageCatalogPlatform imageCatalogPlatform; @@ -299,79 +231,6 @@ static Object[][] failedParamProvider() { }; } - static Object[][] razCloudPlatformDataProvider() { - return new Object[][]{ - // testCaseName cloudPlatform - {"CloudPlatform.AWS multiaz=true", AWS, true}, - {"CloudPlatform.AWS multiaz=false", AWS, false}, - {"CloudPlatform.AZURE multiaz=false", AZURE, false}, - {"CloudPlatform.GCP multiaz=false", GCP, false} - }; - } - - static Object[][] razCloudPlatformAndRuntimeDataProvider() { - return new Object[][]{ - // testCaseName cloudPlatform runtime - {"CloudPlatform.AWS", AWS, "7.2.2"}, - {"CloudPlatform.AZURE", AZURE, "7.2.2"}, - {"CloudPlatform.GCP", GCP, "7.2.17"} - }; - } - - static Object[][] razCloudPlatform710DataProvider() { - return new Object[][]{ - // testCaseName cloudPlatform expectedErrorMsg - {"CloudPlatform.AWS", AWS, - "Provisioning Ranger Raz on Amazon Web Services is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.2 and not 7.1.0"}, - {"CloudPlatform.AZURE", AZURE, - "Provisioning Ranger Raz on Microsoft Azure is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.2 and not 7.1.0"}, - {"CloudPlatform.GCP", GCP, - "Provisioning Ranger Raz on GCP is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.17 and not 7.1.0"} - }; - } - - static Object[][] razCloudPlatform720DataProvider() { - return new Object[][]{ - // testCaseName cloudPlatform expectedErrorMsg - {"CloudPlatform.AWS", AWS, - "Provisioning Ranger Raz on Amazon Web Services is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.2 and not 7.2.0"}, - {"CloudPlatform.AZURE", AZURE, - "Provisioning Ranger Raz on Microsoft Azure is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.2 and not 7.2.0"}, - {"CloudPlatform.GCP", GCP, - "Provisioning Ranger Raz on GCP is only valid for Cloudera Runtime version " + - "greater than or equal to 7.2.17 and not 7.2.0"} - }; - } - - static Object[][] ccmV2Scenarios() { - return new Object[][]{ - // runtime compatible - {null, true}, - {"7.2.0", false}, - {"7.2.1", true}, - {"7.2.5", true}, - {"7.2.6", true}, - {"7.2.7", true}, - }; - } - - static Object[][] ccmV2JumpgateScenarios() { - return new Object[][]{ - // runtime compatible - {null, true}, - {"7.2.0", false}, - {"7.2.1", false}, - {"7.2.5", false}, - {"7.2.6", true}, - {"7.2.7", true}, - }; - } - public static Stream storageBaseLocationsWhiteSpaceValidation() { return Stream.of( Arguments.of(" abfs://myscontainer@mystorage", ValidationResult.State.VALID), @@ -383,14 +242,8 @@ public static Stream storageBaseLocationsWhiteSpaceValidation() { @BeforeEach void initMocks() { - MockitoAnnotations.initMocks(this); lenient().when(platformConfig.getRazSupportedPlatforms()) .thenReturn(List.of(AWS, AZURE, GCP)); - lenient().when(entitlementService.isRazForGcpEnabled(anyString())) - .thenReturn(true); - lenient().when(platformStringTransformer.getPlatformStringForImageCatalog(anyString(), anyBoolean())).thenReturn(imageCatalogPlatform); - lenient().when(externalDatabaseConfigurer.configure(any(CloudPlatform.class), any(), ArgumentMatchers.isNull(), any(SdxDatabaseRequest.class), - any(SdxCluster.class))).thenReturn(new SdxDatabase()); } @Test @@ -535,205 +388,6 @@ void testUpdateRangerRazIsIgnoredIfRangerRazIsInstalledAndFlagAlreadySet() { verify(sdxClusterRepository, times(0)).save(sdxCluster); } - @Test - void testCreateSdxClusterWhenClusterWithSpecifiedNameAlreadyExistShouldThrowClusterAlreadyExistBadRequestException() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); - SdxCluster existing = new SdxCluster(); - existing.setEnvName("envir"); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse( - anyString(), anyString())).thenReturn(Collections.singletonList(existing)); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("SDX cluster exists for environment name: envir", badRequestException.getMessage()); - } - - @Test - void testCreateSdxClusterWithoutCloudStorageShouldThrownBadRequestException() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Cloud storage parameter is required.", badRequestException.getMessage()); - } - - @Test - void testCreateSdxClusterWithoutCloudStorageShouldNotThrownBadRequestExceptionInCaseOfInternal() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", CUSTOM); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - StackV4Request stackV4Request = new StackV4Request(); - ClusterV4Request clusterV4Request = new ClusterV4Request(); - stackV4Request.setCluster(clusterV4Request); - ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); - } - - @Test - void testNullJavaVersionShouldNotOverrideTheVersionInTheInternalStackRequest() throws IOException, TransactionExecutionException { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.1", CUSTOM); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AWS, null); - StackV4Request stackV4Request = new StackV4Request(); - stackV4Request.setJavaVersion(8); - ClusterV4Request clusterV4Request = new ClusterV4Request(); - stackV4Request.setCluster(clusterV4Request); - - ArgumentCaptor sdxClusterArgumentCaptor = ArgumentCaptor.forClass(SdxCluster.class); - when(sdxClusterRepository.save(sdxClusterArgumentCaptor.capture())).thenReturn(mock(SdxCluster.class)); - - ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); - - StackV4Request savedStackV4Request = JsonUtil.readValue(sdxClusterArgumentCaptor.getValue().getStackRequest(), StackV4Request.class); - assertEquals(8, savedStackV4Request.getJavaVersion()); - } - - @Test - void testCreateNOTInternalSdxClusterFromLightDutyTemplateShouldTriggerSdxCreationFlow() throws IOException, TransactionExecutionException { - CrnTestUtil.mockCrnGenerator(regionAwareCrnGenerator); - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, AZURE, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - SdxCluster capturedSdx = captor.getValue(); - assertEquals("tagecske", capturedSdx.getTags().getValue("mytag")); - assertEquals(CLUSTER_NAME, capturedSdx.getClusterName()); - assertEquals(LIGHT_DUTY, capturedSdx.getClusterShape()); - assertEquals("envir", capturedSdx.getEnvName()); - assertEquals("hortonworks", capturedSdx.getAccountId()); - verify(sdxStatusService, times(1)).setStatusForDatalakeAndNotify(DatalakeStatusEnum.REQUESTED, "Datalake requested", createdSdxCluster); - - assertEquals(1L, capturedSdx.getCreated()); - assertFalse(capturedSdx.isCreateDatabase()); - assertTrue(createdSdxCluster.getCrn().matches("crn:cdp:datalake:us-west-1:hortonworks:datalake:.*")); - StackV4Request stackV4Request = JsonUtil.readValue(capturedSdx.getStackRequest(), StackV4Request.class); - - assertEquals(2L, stackV4Request.getInstanceGroups().size()); - - InstanceGroupV4Request core = getGroup(stackV4Request, CORE); - assertEquals(1L, core.getSecurityGroup().getSecurityRules().size()); - assertEquals("0.0.0.0/0", core.getSecurityGroup().getSecurityRules().get(0).getSubnet()); - assertEquals("22", core.getSecurityGroup().getSecurityRules().get(0).getPorts().get(0)); - - InstanceGroupV4Request gateway = getGroup(stackV4Request, GATEWAY); - assertEquals(2L, gateway.getSecurityGroup().getSecurityRules().size()); - assertEquals("0.0.0.0/0", gateway.getSecurityGroup().getSecurityRules().get(0).getSubnet()); - assertEquals("443", gateway.getSecurityGroup().getSecurityRules().get(0).getPorts().get(0)); - assertEquals("0.0.0.0/0", gateway.getSecurityGroup().getSecurityRules().get(1).getSubnet()); - assertEquals("22", gateway.getSecurityGroup().getSecurityRules().get(1).getPorts().get(0)); - - verify(sdxReactorFlowManager).triggerSdxCreation(createdSdxCluster); - } - - @Test - void testCreateNOTInternalSdxClusterFromLightDutyTemplateWhenLocationSpecifiedWithSlashShouldCreateAndSettedUpBaseLocationWithOUTSlash() - throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals("s3a://some/dir", createdSdxCluster.getCloudStorageBaseLocation()); - } - - @Test - void testCreateSdxClusterWithSpotStackRequestContainsRequiredAttributes() throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); - setSpot(sdxClusterRequest); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - // AWS 7.1.0 light duty contains exactly 2 instance groups - assertThat(createdSdxCluster.getStackRequest()).containsSubsequence( - "{\"aws\":{\"spot\":{\"percentage\":100,\"maxPrice\":0.9}}", - "{\"aws\":{\"spot\":{\"percentage\":100,\"maxPrice\":0.9}}"); - } - - @Test - void testCreateSdxClusterWithJavaVersionStackRequestContainsRequiredAttributes() throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - when(virtualMachineConfiguration.getSupportedJavaVersions()).thenReturn(Set.of(11)); - - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); - sdxClusterRequest.setJavaVersion(11); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - - assertThat(createdSdxCluster.getStackRequest()).containsSubsequence("\"javaVersion\":11"); - } - - @Test - void testCreateNOTInternalSdxClusterFromLightDutyTemplateWhenBaseLocationSpecifiedShouldCreateStackRequestWithSettedUpBaseLocation() - throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - //doNothing().when(cloudStorageLocationValidator.validate("s3a://some/dir", )); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, LIGHT_DUTY); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals("s3a://some/dir", createdSdxCluster.getCloudStorageBaseLocation()); - } - @ParameterizedTest @MethodSource("storageBaseLocationsWhiteSpaceValidation") void testValidateBaseLocationWhenWhiteSpaceIsPresent(String input, ValidationResult.State expected) { @@ -741,57 +395,6 @@ void testValidateBaseLocationWhenWhiteSpaceIsPresent(String input, ValidationRes assertEquals(expected, result.getState()); } - @MethodSource("startParamProvider") - void testCreateButEnvInStoppedStatus(EnvironmentStatus environmentStatus, String exceptionMessage) { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); - - DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); - detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); - detailedEnvironmentResponse.setCloudPlatform(AWS.name()); - detailedEnvironmentResponse.setCrn(getCrn()); - detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); - when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); - - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), - "BadRequestException should thrown"); - assertEquals(exceptionMessage, badRequestException.getMessage()); - } - - @ParameterizedTest - @MethodSource("deleteInProgressParamProvider") - void testCreateButEnvInDeleteInProgressPhase(EnvironmentStatus environmentStatus) { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); - DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); - detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); - detailedEnvironmentResponse.setCloudPlatform(AWS.name()); - detailedEnvironmentResponse.setCrn(getCrn()); - detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); - when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); - - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), - "BadRequestException should thrown"); - assertEquals("The environment is in delete in progress phase. Please create a new environment first!", badRequestException.getMessage()); - } - - @ParameterizedTest - @MethodSource("failedParamProvider") - void testCreateButEnvInFailedPhase(EnvironmentStatus environmentStatus) { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); - DetailedEnvironmentResponse detailedEnvironmentResponse = new DetailedEnvironmentResponse(); - detailedEnvironmentResponse.setName(sdxClusterRequest.getEnvironment()); - detailedEnvironmentResponse.setCloudPlatform(AWS.name()); - detailedEnvironmentResponse.setCrn(getCrn()); - detailedEnvironmentResponse.setEnvironmentStatus(environmentStatus); - when(environmentClientService.getByName(anyString())).thenReturn(detailedEnvironmentResponse); - - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)), - "BadRequestException should thrown"); - assertEquals("The environment is in failed phase. Please fix the environment or create a new one first!", badRequestException.getMessage()); - } - @Test void testListSdxClustersWhenEnvironmentNameProvidedAndTwoSdxIsInTheDatabaseShouldListAllSdxClusterWhichIsTwo() { List sdxClusters = List.of(new SdxCluster(), new SdxCluster()); @@ -828,167 +431,6 @@ void testSyncSdxClusterWhenClusterNameSpecifiedShouldCallStackEndpointExactlyOnc verify(stackV4Endpoint, times(1)).sync(eq(0L), eq(CLUSTER_NAME), anyString()); } - @ParameterizedTest(name = "{0}") - @MethodSource("razCloudPlatformDataProvider") - void testSdxCreateRazNotRequestedAndMultiAzRequested(String testCaseName, CloudPlatform cloudPlatform, boolean multiAz) - throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.1.0", LIGHT_DUTY); - withCloudStorage(sdxClusterRequest); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); - sdxClusterRequest.setEnableRangerRaz(false); - sdxClusterRequest.setEnableMultiAz(multiAz); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - SdxCluster capturedSdx = captor.getValue(); - assertFalse(capturedSdx.isRangerRazEnabled()); - assertEquals(multiAz, capturedSdx.isEnableMultiAz()); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("razCloudPlatformAndRuntimeDataProvider") - void testSdxCreateRazEnabled(String testCaseName, CloudPlatform cloudPlatform, String runtime) throws IOException, TransactionExecutionException { - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.1.0/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); - sdxClusterRequest.setEnableRangerRaz(true); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - SdxCluster capturedSdx = captor.getValue(); - assertTrue(capturedSdx.isRangerRazEnabled()); - } - - @Test - void testSdxCreateRazEnabledWithRazNotEnabledCloud() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.2", LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.YARN, null); - sdxClusterRequest.setEnableRangerRaz(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning Ranger Raz is only valid for Amazon Web Services, Microsoft Azure, GCP", badRequestException.getMessage()); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("razCloudPlatform710DataProvider") - void testSdxCreateRazEnabled710Runtime(String testCaseName, CloudPlatform cloudPlatform, String expectedErrorMsg) { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.1.0", LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); - sdxClusterRequest.setEnableRangerRaz(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals(expectedErrorMsg, badRequestException.getMessage()); - } - - @ParameterizedTest(name = "{0}") - @MethodSource("razCloudPlatform720DataProvider") - void testSdxCreateRazEnabled720Runtime(String testCaseName, CloudPlatform cloudPlatform, String expectedErrorMsg) { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.0", LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, cloudPlatform, null); - sdxClusterRequest.setEnableRangerRaz(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals(expectedErrorMsg, badRequestException.getMessage()); - } - - @Test - void testSdxCreateRazEnabledForGcpEntitilementNotEnabled() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.17", LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - when(entitlementService.isRazForGcpEnabled(anyString())).thenReturn(false); - mockEnvironmentCall(sdxClusterRequest, GCP, null); - - sdxClusterRequest.setEnableRangerRaz(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning Ranger Raz on GCP is not enabled for this account", badRequestException.getMessage()); - } - - @Test - void testSdxCreateMediumDutySdx() throws IOException, TransactionExecutionException { - final String runtime = "7.2.7"; - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/medium_duty_ha.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MEDIUM_DUTY_HA); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - SdxCluster capturedSdx = captor.getValue(); - assertEquals(MEDIUM_DUTY_HA, capturedSdx.getClusterShape()); - } - - @Test - void testSdxCreateMediumDutySdx710Runtime() { - final String invalidRuntime = "7.1.0"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(invalidRuntime, MEDIUM_DUTY_HA); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning a Medium Duty SDX shape is only valid for CM version greater than or equal to " - + MEDIUM_DUTY_REQUIRED_VERSION + " and not " + invalidRuntime, badRequestException.getMessage()); - } - - @Test - void testSdxCreateMediumDutySdx720Runtime() { - final String invalidRuntime = "7.2.0"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(invalidRuntime, MEDIUM_DUTY_HA); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning a Medium Duty SDX shape is only valid for CM version greater than or equal to " - + MEDIUM_DUTY_REQUIRED_VERSION + " and not " + invalidRuntime, badRequestException.getMessage()); - } - @Test public void testUpdateRuntimeVersionFromStackResponse() { SdxCluster sdxCluster = getSdxCluster(); @@ -1044,101 +486,6 @@ public void testGetWithEnvironmentCrnsByResourceCrns() { assertEquals(expected, result); } - @Test - void testCreateSdxClusterWithCustomRequestContainsImageInfo() throws Exception { - ImageV4Response imageResponse = getImageResponse(); - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.7/aws/light_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - SdxCustomClusterRequest sdxClusterRequest = createSdxCustomClusterRequest(LIGHT_DUTY, "cdp-default", "imageId_1"); - setSpot(sdxClusterRequest); - withCloudStorage(sdxClusterRequest); - when(imageCatalogService.getImageResponseFromImageRequest(eq(sdxClusterRequest.getImageSettingsV4Request()), any())).thenReturn(imageResponse); - - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, - () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest)); - - SdxCluster createdSdxCluster = result.getLeft(); - StackV4Request stackV4Request = JsonUtil.readValue(createdSdxCluster.getStackRequest(), StackV4Request.class); - assertNotNull(stackV4Request.getImage()); - assertEquals("cdp-default", stackV4Request.getImage().getCatalog()); - assertEquals("imageId_1", stackV4Request.getImage().getId()); - verify(externalDatabaseConfigurer).configure(any(), eq(OS), any(), any(), any()); - } - - @Test - void testCreateInternalSdxClusterWithCustomInstanceGroupShouldFail() { - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - StackV4Request stackV4Request = new StackV4Request(); - ClusterV4Request clusterV4Request = new ClusterV4Request(); - stackV4Request.setCluster(clusterV4Request); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest("7.2.12", CUSTOM); - withCustomInstanceGroups(sdxClusterRequest); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, stackV4Request)); - assertEquals("Custom instance group is not accepted on SDX Internal API.", badRequestException.getMessage()); - } - - @Test - void testCreateSdxClusterWithCustomInstanceGroup() throws Exception { - final String runtime = "7.2.12"; - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String microDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/micro_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(microDutyJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - withRecipe(sdxClusterRequest); - withCustomInstanceGroups(sdxClusterRequest); - RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); - RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); - recipeViewV4Response.setName("post-service-deployment"); - recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); - when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); - when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); - - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); - SdxCluster capturedSdx = captor.getValue(); - assertEquals(MICRO_DUTY, capturedSdx.getClusterShape()); - StackV4Request stackRequest = JsonUtil.readValue(capturedSdx.getStackRequest(), StackV4Request.class); - Optional masterGroup = stackRequest.getInstanceGroups().stream() - .filter(instanceGroup -> "master".equals(instanceGroup.getName())) - .findAny(); - assertTrue(masterGroup.isPresent()); - assertEquals("verylarge", masterGroup.get().getTemplate().getInstanceType()); - Optional idbrokerGroup = stackRequest.getInstanceGroups().stream() - .filter(instanceGroup -> "idbroker".equals(instanceGroup.getName())) - .findAny(); - assertTrue(idbrokerGroup.isPresent()); - assertEquals("notverylarge", idbrokerGroup.get().getTemplate().getInstanceType()); - } - private void setSpot(SdxClusterRequestBase sdxClusterRequest) { SdxAwsRequest aws = new SdxAwsRequest(); SdxAwsSpotParameters spot = new SdxAwsSpotParameters(); @@ -1661,170 +1008,6 @@ void testSdxResizeClusterWithNoEntitlement() { assertEquals("Resizing of the data lake is not supported", badRequestException.getMessage()); } - @ParameterizedTest(name = "Runtime {0} is compatible with CCMv2 = {1}") - @MethodSource("ccmV2Scenarios") - void testCcmV2VersionChecker(String runtime, boolean compatible) throws IOException { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.10/aws/light_duty.json"); - lenient().when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.MOCK, Tunnel.CCMV2); - if (!compatible) { - assertThatThrownBy(() -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))) - .isInstanceOf(BadRequestException.class) - .hasMessage(String.format("Runtime version %s does not support Cluster Connectivity Manager. " - + "Please try creating a datalake with runtime version at least %s.", runtime, CCMV2_REQUIRED_VERSION)); - } else { - ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - } - } - - @ParameterizedTest(name = "Runtime {0} is compatible with CCMv2JumpGate = {1}") - @MethodSource("ccmV2JumpgateScenarios") - void testCcmV2JumpgateVersionChecker(String runtime, boolean compatible) throws IOException { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, LIGHT_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - String lightDutyJson = FileReaderUtils.readFileFromClasspath("/duties/7.2.10/aws/light_duty.json"); - lenient().when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(lightDutyJson, StackV4Request.class)); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.MOCK, Tunnel.CCMV2_JUMPGATE); - if (!compatible) { - assertThatThrownBy(() -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))) - .isInstanceOf(BadRequestException.class) - .hasMessage(String.format("Runtime version %s does not support Cluster Connectivity Manager. " - + "Please try creating a datalake with runtime version at least %s.", runtime, CCMV2_JUMPGATE_REQUIRED_VERSION)); - } else { - ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - } - } - - @Test - void testCreateMicroDuty() throws IOException, TransactionExecutionException { - final String runtime = "7.2.12"; - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String microDutyJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/micro_duty.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(microDutyJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - withRecipe(sdxClusterRequest); - RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); - RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); - recipeViewV4Response.setName("post-service-deployment"); - recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); - when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); - when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); - when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, AWS, null); - when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); - SdxCluster capturedSdx = captor.getValue(); - assertEquals(MICRO_DUTY, capturedSdx.getClusterShape()); - } - - @Test - void testCreateMicroDutyWrongVersion() { - final String runtime = "7.2.11"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AZURE, null); - when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning a Micro Duty SDX shape is only valid for CM version greater than or equal to 7.2.12 and not 7.2.11", - badRequestException.getMessage()); - } - - @Test - void testCreateMicroDutyNoEntitlement() { - final String runtime = "7.2.12"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, MICRO_DUTY); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AZURE, null); - when(entitlementService.microDutySdxEnabled(anyString())).thenReturn(false); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals(String.format("Provisioning a micro duty data lake cluster is not enabled for %s. ", AZURE.name()) + - "Contact Cloudera support to enable CDP_MICRO_DUTY_SDX entitlement for the account.", badRequestException.getMessage()); - } - - @Test - void testCreateEnterpriseDatalake() throws IOException, TransactionExecutionException { - final String runtime = "7.2.17"; - when(transactionService.required(isA(Supplier.class))).thenAnswer(invocation -> invocation.getArgument(0, Supplier.class).get()); - String enterpriseJson = FileReaderUtils.readFileFromClasspath("/duties/" + runtime + "/aws/enterprise.json"); - when(cdpConfigService.getConfigForKey(any())).thenReturn(JsonUtil.readValue(enterpriseJson, StackV4Request.class)); - when(sdxReactorFlowManager.triggerSdxCreation(any())).thenReturn(new FlowIdentifier(FlowType.FLOW, "FLOW_ID")); - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - withCloudStorage(sdxClusterRequest); - withRecipe(sdxClusterRequest); - RecipeViewV4Responses recipeViewV4Responses = new RecipeViewV4Responses(); - RecipeViewV4Response recipeViewV4Response = new RecipeViewV4Response(); - recipeViewV4Response.setName("post-service-deployment"); - recipeViewV4Responses.setResponses(List.of(recipeViewV4Response)); - when(recipeV4Endpoint.listInternal(anyLong(), anyString())).thenReturn(recipeViewV4Responses); - when(regionAwareInternalCrnGenerator.getInternalCrnForServiceAsString()).thenReturn("crn:cdp:freeipa:us-west-1:altus:user:__internal__actor__"); - when(regionAwareInternalCrnGeneratorFactory.iam()).thenReturn(regionAwareInternalCrnGenerator); - long id = 10L; - when(sdxClusterRepository.save(any(SdxCluster.class))).thenAnswer(invocation -> { - SdxCluster sdxWithId = invocation.getArgument(0, SdxCluster.class); - sdxWithId.setId(id); - return sdxWithId; - }); - when(clock.getCurrentTimeMillis()).thenReturn(1L); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AWS, null); - when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(false); - Pair result = ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null)); - SdxCluster createdSdxCluster = result.getLeft(); - assertEquals(id, createdSdxCluster.getId()); - ArgumentCaptor captor = ArgumentCaptor.forClass(SdxCluster.class); - verify(sdxClusterRepository, times(1)).save(captor.capture()); - verify(recipeV4Endpoint, times(1)).listInternal(anyLong(), anyString()); - SdxCluster capturedSdx = captor.getValue(); - assertEquals(ENTERPRISE, capturedSdx.getClusterShape()); - } - - @Test - void testCreateEnterpriseDatalakeWrongVersion() { - final String runtime = "7.2.11"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, CloudPlatform.AZURE, null); - when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(false); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning an Enterprise SDX shape is only valid for CM version greater than or equal to 7.2.17 and not 7.2.11", - badRequestException.getMessage()); - } - - @Test - void testCreateEnterpriseDatalakeWithDisabledEntitlement() { - final String runtime = "7.2.17"; - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(runtime, ENTERPRISE); - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse(anyString(), anyString())).thenReturn(new ArrayList<>()); - mockEnvironmentCall(sdxClusterRequest, AZURE, null); - when(entitlementService.enterpriseSdxDisabled(anyString())).thenReturn(true); - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Provisioning an enterprise data lake cluster is disabled. " + - "Contact Cloudera support to enable this scale for the account.", badRequestException.getMessage()); - } - @Test void testAttachRecipe() { SdxCluster sdxCluster = setupRecipeTest(); @@ -1974,19 +1157,4 @@ public void testUpdateSaltThrowsBadRequestWhenDatalakeNotAvailable(DatalakeStatu verifyNoInteractions(sdxReactorFlowManager); assertEquals(String.format("SaltStack update cannot be initiated as datalake 'sdx-cluster-name' is currently in '%s' state.", status), ex.getMessage()); } - - @Test - void testCreateSdxClusterFialsInCaseOfForcedJavaVersionIsNotSupportedByTheVirtualMachineConfiguration() { - SdxClusterRequest sdxClusterRequest = createSdxClusterRequest(null, MEDIUM_DUTY_HA); - sdxClusterRequest.setJavaVersion(11); - - when(sdxClusterRepository.findByAccountIdAndEnvNameAndDeletedIsNullAndDetachedIsFalse( - anyString(), anyString())).thenReturn(Collections.emptyList()); - when(virtualMachineConfiguration.getSupportedJavaVersions()).thenReturn(Collections.emptySet()); - - BadRequestException badRequestException = assertThrows(BadRequestException.class, - () -> ThreadBasedUserCrnProvider.doAs(USER_CRN, () -> - underTest.createSdx(USER_CRN, CLUSTER_NAME, sdxClusterRequest, null))); - assertEquals("Java version 11 is not supported.", badRequestException.getMessage()); - } -} \ No newline at end of file +} diff --git a/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceValidateRuntimeAndImageTest.java b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceValidateRuntimeAndImageTest.java index a50ad442097..d5cad636b9c 100644 --- a/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceValidateRuntimeAndImageTest.java +++ b/datalake/src/test/java/com/sequenceiq/datalake/service/sdx/SdxServiceValidateRuntimeAndImageTest.java @@ -33,6 +33,7 @@ void setUp() { environment = new DetailedEnvironmentResponse(); environment.setCloudPlatform("AWS"); imageSettingsV4Request = new ImageSettingsV4Request(); + imageSettingsV4Request.setId("image-id"); imageV4Response = new ImageV4Response(); } diff --git a/environment/src/main/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorService.java b/environment/src/main/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorService.java index b39a483507d..af1c099b694 100644 --- a/environment/src/main/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorService.java +++ b/environment/src/main/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorService.java @@ -268,6 +268,9 @@ public ValidationResult validateFreeIpaCreation(FreeIpaCreationDto freeIpaCreati validationResultBuilder.error( String.format("Single instance FreeIpa spot percentage must be either %d or %d.", ALL_ON_DEMAND_PERCENTAGE, ALL_SPOT_PERCENTAGE)); } + if (StringUtils.isNoneBlank(freeIpaCreation.getImageId(), freeIpaCreation.getImageOs())) { + validationResultBuilder.error("FreeIpa deployment requests can not have both image id and image os parameters set."); + } return validationResultBuilder.build(); } diff --git a/environment/src/test/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorServiceTest.java b/environment/src/test/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorServiceTest.java index 54041b28466..7becb46c15f 100644 --- a/environment/src/test/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorServiceTest.java +++ b/environment/src/test/java/com/sequenceiq/environment/environment/validation/EnvironmentValidatorServiceTest.java @@ -434,6 +434,19 @@ void testValidateWhenRequestedInstanceCountEqualsOrMoreThanTheMinimumThreshold() assertFalse(validationResult.hasError()); } + @Test + void testValidateWhenRequestedFreeipaHasImageIdAndImageOs() { + FreeIpaCreationDto freeIpaCreationDto = FreeIpaCreationDto.builder(1) + .withImageId("id") + .withImageOs("os") + .build(); + + ValidationResult validationResult = underTest.validateFreeIpaCreation(freeIpaCreationDto); + assertTrue(validationResult.hasError()); + assertEquals("FreeIpa deployment requests can not have both image id and image os parameters set.", + validationResult.getErrors().get(0)); + } + private Environment aValidEnvirontmentWithParent() { Environment parentEnvironment = new Environment(); parentEnvironment.setCloudPlatform(CloudPlatform.AWS.name());