CB-22446 Backport DL/DH os selection to 2.73

core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/request/image/ImageSettingsV4Request.java

@@ -20,7 +20,7 @@ public class ImageSettingsV4Request implements JsonEntity {
     @ApiModelProperty(StackModelDescription.IMAGE_ID)
     private String id;
 
-    @ApiModelProperty(StackModelDescription.OS_TYPE)
+    @ApiModelProperty(StackModelDescription.IMAGE_OS)
     private String os;
 
     public String getCatalog() {

core-api/src/main/java/com/sequenceiq/cloudbreak/api/endpoint/v4/stacks/response/image/StackImageV4Response.java

@@ -3,6 +3,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.sequenceiq.cloudbreak.doc.ModelDescriptions;
 import com.sequenceiq.cloudbreak.doc.ModelDescriptions.ImageModelDescription;
 import com.sequenceiq.common.model.JsonEntity;
 
@@ -25,6 +26,9 @@ public class StackImageV4Response implements JsonEntity {
     @ApiModelProperty(ImageModelDescription.IMAGE_CATALOG_NAME)
     private String catalogName;
 
+    @ApiModelProperty(ModelDescriptions.StackModelDescription.IMAGE_OS)
+    private String os;
+
     public String getName() {
         return name;
     }
@@ -56,4 +60,12 @@ public String getCatalogName() {
     public void setCatalogName(String catalogName) {
         this.catalogName = catalogName;
     }
+
+    public String getOs() {
+        return os;
+    }
+
+    public void setOs(String os) {
+        this.os = os;
+    }
 }

core-api/src/main/java/com/sequenceiq/cloudbreak/doc/ModelDescriptions.java

@@ -219,7 +219,7 @@ public static class StackModelDescription {
         public static final String IMAGE_SETTINGS = "settings for custom images";
         public static final String IMAGE_CATALOG = "custom image catalog URL";
         public static final String IMAGE_ID = "virtual machine image id from ImageCatalog, machines of the cluster will be started from this image";
-        public static final String OS_TYPE = "os type of the image, this property is only considered when no specific image id is provided";
+        public static final String IMAGE_OS = "os of the image, this property is only considered when no specific image id is provided";
         public static final String INSTANCE_GROUP_ADJUSTMENT = "instance group adjustment";
         public static final String TAGS = "stack related tags";
         public static final String APPLICATION_TAGS = "stack related application tags";

core-api/src/main/java/com/sequenceiq/distrox/api/v1/distrox/model/image/DistroXImageV1Request.java

@@ -21,6 +21,9 @@ public class DistroXImageV1Request implements Serializable {
     @ApiModelProperty(StackModelDescription.IMAGE_ID)
     private String id;
 
+    @ApiModelProperty(StackModelDescription.IMAGE_OS)
+    private String os;
+
     public String getCatalog() {
         return catalog;
     }
@@ -36,4 +39,12 @@ public String getId() {
     public void setId(String id) {
         this.id = id;
     }
+
+    public String getOs() {
+        return os;
+    }
+
+    public void setOs(String os) {
+        this.os = os;
+    }
 }

core/src/main/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogService.java

@@ -291,7 +291,7 @@ public StatedImage getLatestBaseImageDefaultPreferred(ImageFilter imageFilter, P
         List<Image> baseImages = statedImages.getImages().getBaseImages();
         Optional<Image> defaultBaseImage = getLatestImageDefaultPreferred(baseImages);
         if (defaultBaseImage.isEmpty()) {
-            throw new CloudbreakImageNotFoundException(baseImageNotFoundErrorMessage(platform, imageFilter.getImageCatalog()));
+            throw new CloudbreakImageNotFoundException(baseImageNotFoundErrorMessage(platform, imageFilter));
         }
         return statedImage(defaultBaseImage.get(), statedImages.getImageCatalogUrl(), statedImages.getImageCatalogName());
     }
@@ -305,8 +305,7 @@ public StatedImage getImagePrewarmedDefaultPreferred(ImageFilter imageFilter, Pr
                     .map(ImageCatalogPlatform::nameToLowerCase)
                     .findFirst();
             String platform = firstImage.orElse("");
-            throw new CloudbreakImageNotFoundException(
-                    imageNotFoundErrorMessage(platform, imageFilter.getClusterVersion(), imageFilter.getImageCatalog()));
+            throw new CloudbreakImageNotFoundException(imageNotFoundErrorMessage(platform, imageFilter));
         }
         return statedImage(selectedImage.get(), statedImages.getImageCatalogUrl(), statedImages.getImageCatalogName());
     }
@@ -330,14 +329,22 @@ private List<Image> filterImagesByRuntimeVersion(String clusterVersion, List<Ima
         return matchingVersionImages;
     }
 
-    private String baseImageNotFoundErrorMessage(String platform, ImageCatalog imageCatalog) {
-        return String.format("Could not find any base image for platform '%s' and Cloudbreak version '%s' in '%s' image catalog.",
-                platform, cbVersion, Optional.ofNullable(imageCatalog).map(ImageCatalog::getName).orElse(null));
+    private String baseImageNotFoundErrorMessage(String platform, ImageFilter imageFilter) {
+        return String.format("Could not find any base image for platform '%s', os '%s' and Cloudbreak version '%s' in '%s' image catalog.",
+                platform, getOses(imageFilter), cbVersion, getImageCatalogName(imageFilter));
+    }
+
+    private String imageNotFoundErrorMessage(String platform, ImageFilter imageFilter) {
+        return String.format("Could not find any image for platform '%s', os '%s', runtime '%s' and Cloudbreak version '%s' in '%s' image catalog.",
+                platform, getOses(imageFilter), imageFilter.getClusterVersion(), cbVersion, getImageCatalogName(imageFilter));
+    }
+
+    private String getOses(ImageFilter imageFilter) {
+        return Optional.ofNullable(imageFilter.getOperatingSystems()).stream().flatMap(Set::stream).collect(Collectors.joining(","));
     }
 
-    private String imageNotFoundErrorMessage(String platform, String runtime, ImageCatalog imageCatalog) {
-        return String.format("Could not find any image for platform '%s', runtime '%s' and Cloudbreak version '%s' in '%s' image catalog.",
-                platform, runtime, cbVersion, Optional.ofNullable(imageCatalog).map(ImageCatalog::getName).orElse(null));
+    private String getImageCatalogName(ImageFilter imageFilter) {
+        return Optional.ofNullable(imageFilter.getImageCatalog()).map(ImageCatalog::getName).orElse(null);
     }
 
     public StatedImages getImages(Long workspaceId, String imageCatalogName, ImageCatalogPlatform provider) throws CloudbreakImageCatalogException {

core/src/main/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverter.java

@@ -12,13 +12,15 @@ public ImageSettingsV4Request convert(DistroXImageV1Request source) {
         ImageSettingsV4Request response = new ImageSettingsV4Request();
         response.setCatalog(source.getCatalog());
         response.setId(source.getId());
+        response.setOs(source.getOs());
         return response;
     }
 
     public DistroXImageV1Request convert(ImageSettingsV4Request source) {
         DistroXImageV1Request response = new DistroXImageV1Request();
         response.setCatalog(source.getCatalog());
         response.setId(source.getId());
+        response.setOs(source.getOs());
         return response;
     }
 

core/src/main/java/com/sequenceiq/distrox/v1/distrox/service/DistroXService.java

@@ -8,6 +8,7 @@
 
 import javax.inject.Inject;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 
@@ -21,6 +22,7 @@
 import com.sequenceiq.cloudbreak.structuredevent.CloudbreakRestRequestThreadLocalService;
 import com.sequenceiq.cloudbreak.workspace.model.Workspace;
 import com.sequenceiq.distrox.api.v1.distrox.model.DistroXV1Request;
+import com.sequenceiq.distrox.api.v1.distrox.model.image.DistroXImageV1Request;
 import com.sequenceiq.distrox.v1.distrox.StackOperations;
 import com.sequenceiq.distrox.v1.distrox.converter.DistroXV1RequestToStackV4RequestConverter;
 import com.sequenceiq.distrox.v1.distrox.fedramp.FedRampModificationService;
@@ -86,6 +88,10 @@ private void validate(DistroXV1Request request) {
         if (!sdxCrnsWithAvailability.stream().map(Pair::getValue).allMatch(isSdxAvailable())) {
             throw new BadRequestException("Data Lake stacks of environment should be available.");
         }
+        DistroXImageV1Request imageRequest = request.getImage();
+        if (imageRequest != null && StringUtils.isNoneBlank(imageRequest.getId(), imageRequest.getOs())) {
+            throw new BadRequestException("Image request can not have both image id and os parameters set.");
+        }
     }
 
     private Predicate<StatusCheckResult> isSdxAvailable() {

core/src/test/java/com/sequenceiq/cloudbreak/service/image/ImageCatalogServiceDefaultNotFoundTest.java

@@ -111,7 +111,8 @@ public void testGetDefaultImageShouldThrowNotFoundException() throws Exception {
         try {
             underTest.getImagePrewarmedDefaultPreferred(imageFilter, image -> true);
         } catch (CloudbreakImageNotFoundException exception) {
-            Assertions.assertEquals("Could not find any image for platform 'gcp', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.",
+            Assertions.assertEquals(
+                    "Could not find any image for platform 'gcp', os 'notimportant', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.",
                     exception.getMessage());
         }
         verify(providerSpecificImageFilter, never()).filterImages(any(), anyList());
@@ -126,11 +127,13 @@ public void testGetDefaultImageShouldThrowNotFoundException2() throws Exception
         when(cloudbreakVersionListProvider.getVersions(any())).thenReturn(catalog.getVersions().getCloudbreakVersions());
         when(imageCatalog.getImageCatalogUrl()).thenReturn(DEFAULT_CDH_IMAGE_CATALOG);
 
-        ImageFilter imageFilter = new ImageFilter(imageCatalog, Set.of(imageCatalogPlatform("aws")), "2.6", true, Set.of("centos7", "amazonlinux2"), null);
+        ImageFilter imageFilter = new ImageFilter(imageCatalog, Set.of(imageCatalogPlatform("aws")), "2.6", true, Set.of("centos7"), null);
         try {
             underTest.getImagePrewarmedDefaultPreferred(imageFilter, image -> true);
         } catch (CloudbreakImageNotFoundException exception) {
-            Assertions.assertEquals("Could not find any image for platform 'aws', runtime 'null' and Cloudbreak version '5.0.0' in 'null' image catalog.",
+            Assertions.assertEquals(
+                    "Could not find any image for platform 'aws', os 'centos7', runtime 'null' " +
+                            "and Cloudbreak version '5.0.0' in 'null' image catalog.",
                     exception.getMessage());
         }
         verify(providerSpecificImageFilter, times(3)).filterImages(eq(Set.of(imageCatalogPlatform(PROVIDERS[0]))), anyList());

core/src/test/java/com/sequenceiq/distrox/v1/distrox/converter/DistroXImageToImageSettingsConverterTest.java

@@ -23,25 +23,29 @@ void testConvertDistroXImageV1RequestToImageSettingsV4Request() {
         DistroXImageV1Request input = new DistroXImageV1Request();
         input.setCatalog("someCatalog");
         input.setId("someId");
+        input.setOs("someOs");
 
         ImageSettingsV4Request result = underTest.convert(input);
 
         assertNotNull(result);
         assertEquals(input.getCatalog(), result.getCatalog());
         assertEquals(input.getId(), result.getId());
+        assertEquals(input.getOs(), result.getOs());
     }
 
     @Test
     void testConvertImageSettingsV4RequestToDistroXImageV1Request() {
         ImageSettingsV4Request input = new ImageSettingsV4Request();
         input.setCatalog("someCatalog");
         input.setId("someId");
+        input.setOs("someOs");
 
         DistroXImageV1Request result = underTest.convert(input);
 
         assertNotNull(result);
         assertEquals(input.getCatalog(), result.getCatalog());
         assertEquals(input.getId(), result.getId());
+        assertEquals(input.getOs(), result.getOs());
     }
 
 }

core/src/test/java/com/sequenceiq/distrox/v1/distrox/service/DistroXServiceTest.java

@@ -2,6 +2,7 @@
 
 import static com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus.AVAILABLE;
 import static com.sequenceiq.environment.api.v1.environment.model.response.EnvironmentStatus.START_DATAHUB_STARTED;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -42,6 +43,7 @@
 import com.sequenceiq.cloudbreak.workspace.model.Tenant;
 import com.sequenceiq.cloudbreak.workspace.model.Workspace;
 import com.sequenceiq.distrox.api.v1.distrox.model.DistroXV1Request;
+import com.sequenceiq.distrox.api.v1.distrox.model.image.DistroXImageV1Request;
 import com.sequenceiq.distrox.v1.distrox.StackOperations;
 import com.sequenceiq.distrox.v1.distrox.converter.DistroXV1RequestToStackV4RequestConverter;
 import com.sequenceiq.distrox.v1.distrox.fedramp.FedRampModificationService;
@@ -239,14 +241,17 @@ public void testIfDlIsNotExists() {
         DetailedEnvironmentResponse envResponse = new DetailedEnvironmentResponse();
         envResponse.setEnvironmentStatus(AVAILABLE);
         envResponse.setCrn("crn");
+        envResponse.setName(envName);
         DescribeFreeIpaResponse freeipa = new DescribeFreeIpaResponse();
         freeipa.setAvailabilityStatus(AvailabilityStatus.AVAILABLE);
         freeipa.setStatus(com.sequenceiq.freeipa.api.v1.freeipa.stack.model.common.Status.AVAILABLE);
         when(freeipaClientService.getByEnvironmentCrn("crn")).thenReturn(freeipa);
         when(environmentClientService.getByName(envName)).thenReturn(envResponse);
         when(platformAwareSdxConnector.listSdxCrns(any(), any())).thenReturn(Set.of());
 
-        assertThrows(BadRequestException.class, () -> underTest.post(request));
+        assertThatThrownBy(() -> underTest.post(request))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessage("Data Lake stack cannot be found for environment CRN: %s (%s)", envName, envResponse.getCrn());
 
         verify(platformAwareSdxConnector).listSdxCrns(any(), any());
         verifyNoMoreInteractions(platformAwareSdxConnector);
@@ -269,7 +274,9 @@ public void testIfDlIsNotRunning() {
         when(platformAwareSdxConnector.listSdxCrnsWithAvailability(any(), any(), any()))
                 .thenReturn(Set.of(Pair.of(DATALAKE_CRN, StatusCheckResult.NOT_AVAILABLE)));
 
-        assertThrows(BadRequestException.class, () -> underTest.post(request));
+        assertThatThrownBy(() -> underTest.post(request))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessage("Data Lake stacks of environment should be available.");
 
         verify(platformAwareSdxConnector).listSdxCrns(any(), any());
         verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any());
@@ -333,6 +340,40 @@ public void testIfDlRollingUpgradeInProgress() {
         verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any());
     }
 
+    @Test
+    void testIfImageIdAndOsBothSet() {
+        String envName = "someAwesomeEnvironment";
+        DistroXV1Request request = new DistroXV1Request();
+        request.setEnvironmentName(envName);
+        DistroXImageV1Request imageRequest = new DistroXImageV1Request();
+        imageRequest.setId("id");
+        imageRequest.setOs("os");
+        request.setImage(imageRequest);
+        DetailedEnvironmentResponse envResponse = new DetailedEnvironmentResponse();
+        envResponse.setEnvironmentStatus(AVAILABLE);
+        envResponse.setCrn("crn");
+        DescribeFreeIpaResponse freeipa = new DescribeFreeIpaResponse();
+        freeipa.setAvailabilityStatus(AvailabilityStatus.AVAILABLE);
+        freeipa.setStatus(com.sequenceiq.freeipa.api.v1.freeipa.stack.model.common.Status.AVAILABLE);
+        Workspace workspace = new Workspace();
+        Tenant tenant = new Tenant();
+        tenant.setName("test");
+        workspace.setTenant(tenant);
+        when(workspaceService.getForCurrentUser()).thenReturn(workspace);
+        when(freeipaClientService.getByEnvironmentCrn("crn")).thenReturn(freeipa);
+        when(environmentClientService.getByName(envName)).thenReturn(envResponse);
+        when(platformAwareSdxConnector.listSdxCrns(any(), any())).thenReturn(Set.of(DATALAKE_CRN));
+        when(platformAwareSdxConnector.listSdxCrnsWithAvailability(any(), any(), any()))
+                .thenReturn(Set.of(Pair.of(DATALAKE_CRN, StatusCheckResult.AVAILABLE)));
+
+        assertThatThrownBy(() -> underTest.post(request))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessage("Image request can not have both image id and os parameters set.");
+
+        verify(platformAwareSdxConnector).listSdxCrns(any(), any());
+        verify(platformAwareSdxConnector).listSdxCrnsWithAvailability(any(), any(), any());
+    }
+
     private static VerificationMode calledOnce() {
         return times(1);
     }

datalake-api/src/main/java/com/sequenceiq/sdx/api/model/ModelDescriptions.java

@@ -92,6 +92,8 @@ public class ModelDescriptions {
 
     public static final String RUNTIME_VERSION = "Runtime version.";
 
+    public static final String OS = "Operating system.";
+
     public static final String DEFAULT_RUNTIME_VERSION = "Default runtime version.";
 
     public static final String LOCK_COMPONENTS = "Option to lock components during the upgrade.";

datalake-api/src/main/java/com/sequenceiq/sdx/api/model/SdxClusterRequest.java

@@ -16,6 +16,9 @@ public class SdxClusterRequest extends SdxClusterRequestBase {
     @ApiModelProperty(ModelDescriptions.RUNTIME_VERSION)
     private String runtime;
 
+    @ApiModelProperty(ModelDescriptions.OS)
+    private String os;
+
     @ApiModelProperty(ModelDescriptions.RECIPES)
     private Set<SdxRecipe> recipes;
 
@@ -27,6 +30,14 @@ public void setRuntime(String runtime) {
         this.runtime = runtime;
     }
 
+    public String getOs() {
+        return os;
+    }
+
+    public void setOs(String os) {
+        this.os = os;
+    }
+
     public Set<SdxRecipe> getRecipes() {
         return recipes;
     }
@@ -39,6 +50,7 @@ public void setRecipes(Set<SdxRecipe> recipes) {
     public String toString() {
         return "SdxClusterRequest{" +
                 "runtime='" + runtime + '\'' +
+                ", os='" + os + '\'' +
                 ", recipes=" + recipes +
                 "} " + super.toString();
     }

datalake/src/main/java/com/sequenceiq/datalake/service/imagecatalog/ImageCatalogService.java

@@ -46,7 +46,7 @@ public String getResourceCrnByResourceName(String resourceName) {
     }
 
     public ImageV4Response getImageResponseFromImageRequest(ImageSettingsV4Request imageSettingsV4Request, ImageCatalogPlatform imageCatalogPlatform) {
-        if (imageSettingsV4Request != null) {
+        if (imageSettingsV4Request != null && !StringUtils.isBlank(imageSettingsV4Request.getId())) {
             List<ImageV4Response> images = getImagesMatchingRequest(imageSettingsV4Request);
             if (images != null) {
                 String providerName = imageCatalogPlatform.nameToLowerCase();