diff --git a/core/src/main/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunner.java b/core/src/main/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunner.java index 1f408725b95..2aacc692205 100644 --- a/core/src/main/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunner.java +++ b/core/src/main/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunner.java @@ -816,9 +816,9 @@ private Map createLoadBalancerProperties(StackDto stackDto) { private List> getFrontendMap(Set loadBalancers) { return loadBalancers.stream() - .filter(lb -> isNotEmpty(lb.getIp())) - .map(lb -> Map.of("type", lb.getType().name(), "ip", lb.getIp())) - .collect(Collectors.toList()); + .filter(lb -> isNotEmpty(lb.getIp())) + .map(lb -> Map.of("type", lb.getType().name(), "ip", lb.getIp())) + .collect(Collectors.toList()); } private boolean isFloatingIpEnabled(Set loadBalancers) { @@ -897,6 +897,10 @@ private List getRangerFqdn(StackDto stackDto, String primaryGatewayFqdn, List hosts = rangerLocations.stream() .filter(s -> s.contains(gatewayGroupName)) .collect(Collectors.toList()); + // If the ranger is not on Gateway node we have to expose other node. + if (CollectionUtils.isEmpty(hosts)) { + return asList(rangerLocations.iterator().next()); + } return hosts; } return rangerLocations.contains(primaryGatewayFqdn) ? asList(primaryGatewayFqdn) : asList(rangerLocations.iterator().next()); diff --git a/core/src/test/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunnerTest.java b/core/src/test/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunnerTest.java index a85ef016a8d..76a3e320530 100644 --- a/core/src/test/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunnerTest.java +++ b/core/src/test/java/com/sequenceiq/cloudbreak/core/bootstrap/service/host/ClusterHostServiceRunnerTest.java @@ -51,6 +51,7 @@ import com.sequenceiq.cloudbreak.TestUtil; import com.sequenceiq.cloudbreak.api.endpoint.v4.common.StackType; import com.sequenceiq.cloudbreak.api.endpoint.v4.database.base.DatabaseType; +import com.sequenceiq.cloudbreak.api.endpoint.v4.stacks.base.InstanceMetadataType; import com.sequenceiq.cloudbreak.api.service.ExposedService; import com.sequenceiq.cloudbreak.api.service.ExposedServiceCollector; import com.sequenceiq.cloudbreak.auth.CMLicenseParser; @@ -122,6 +123,7 @@ import com.sequenceiq.cloudbreak.util.NodesUnreachableException; import com.sequenceiq.cloudbreak.util.StackUtil; import com.sequenceiq.cloudbreak.view.ClusterView; +import com.sequenceiq.cloudbreak.view.InstanceGroupView; import com.sequenceiq.cloudbreak.view.InstanceMetadataView; import com.sequenceiq.cloudbreak.view.StackView; import com.sequenceiq.common.api.type.LoadBalancerType; @@ -728,6 +730,123 @@ private void setupMocksForRunClusterServices() { when(stackToTemplatePreparationObjectConverter.convert(any())).thenReturn(templatePreparationObject); } + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + void testEDLRangerToplogy() throws IOException, CloudbreakOrchestratorFailedException { + setupMocksForRunClusterServices(); + Set nodes = Sets.newHashSet(node("fqdn1"), node("fqdn2"), node("fqdn3"), + node("gateway1"), node("master1")); + when(stackUtil.collectNodes(any())).thenReturn(nodes); + when(exposedServiceCollector.getAllServiceNames()).thenReturn(Set.of("RANGER")); + Map> locations = new HashMap<>(); + locations.put("RANGER", List.of("master1", "master2")); + when(componentLocator.getComponentLocation(any(), any())).thenReturn(locations); + StackView stackViewMock = mock(StackView.class); + when(stackViewMock.getId()).thenReturn(STACK_ID); + ReflectionTestUtils.setField(stack, "stack", stackViewMock); + InstanceGroupView instanceGroupView = mock(InstanceGroupView.class); + InstanceMetadataView instanceMetadataView = mock(InstanceMetadataView.class); + when(instanceMetadataView.getInstanceMetadataType()).thenReturn(InstanceMetadataType.GATEWAY_PRIMARY); + when(instanceGroupView.getGroupName()).thenReturn("GATEWAY"); + Map instanceGroups = new HashMap<>(); + InstanceGroupDto instanceGroupDto = new InstanceGroupDto(instanceGroupView, List.of(instanceMetadataView)); + instanceGroups.put("GATEWAY", instanceGroupDto); + String edlBP = FileReaderUtils.readFileFromClasspath("cdp-sdx-enterprise.bp"); + Blueprint bp = new Blueprint(); + bp.setBlueprintText(edlBP); + ReflectionTestUtils.setField(stack, "instanceGroups", instanceGroups); + ReflectionTestUtils.setField(stack, "blueprint", bp); + underTest.redeployGatewayPillarOnly(stack); + ArgumentCaptor saltConfigCaptor = ArgumentCaptor.forClass(SaltConfig.class); + verify(hostOrchestrator).uploadGatewayPillar(any(), allNodesCaptor.capture(), any(), saltConfigCaptor.capture()); + Set allNodes = allNodesCaptor.getValue(); + assertEquals(5, allNodes.size()); + SaltConfig saltConfig = saltConfigCaptor.getValue(); + SaltPillarProperties gatewayPillarProperties = saltConfig.getServicePillarConfig().get("gateway"); + Map props = (HashMap) gatewayPillarProperties.getProperties().get("gateway"); + Map> location = (HashMap>) props.get("location"); + List ranger = location.get("RANGER"); + assertTrue(ranger.contains("master1")); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + void testMDRangerToplogy() throws IOException, CloudbreakOrchestratorFailedException { + setupMocksForRunClusterServices(); + Set nodes = Sets.newHashSet(node("fqdn1"), node("fqdn2"), node("fqdn3"), + node("gateway1"), node("master1")); + when(stackUtil.collectNodes(any())).thenReturn(nodes); + when(exposedServiceCollector.getAllServiceNames()).thenReturn(Set.of("RANGER")); + Map> locations = new HashMap<>(); + locations.put("RANGER", List.of("master1", "master2")); + when(componentLocator.getComponentLocation(any(), any())).thenReturn(locations); + StackView stackViewMock = mock(StackView.class); + when(stackViewMock.getId()).thenReturn(STACK_ID); + ReflectionTestUtils.setField(stack, "stack", stackViewMock); + InstanceGroupView instanceGroupView = mock(InstanceGroupView.class); + InstanceMetadataView instanceMetadataView = mock(InstanceMetadataView.class); + when(instanceMetadataView.getInstanceMetadataType()).thenReturn(InstanceMetadataType.GATEWAY_PRIMARY); + when(instanceGroupView.getGroupName()).thenReturn("GATEWAY"); + Map instanceGroups = new HashMap<>(); + InstanceGroupDto instanceGroupDto = new InstanceGroupDto(instanceGroupView, List.of(instanceMetadataView)); + instanceGroups.put("GATEWAY", instanceGroupDto); + String mediumDutyBP = FileReaderUtils.readFileFromClasspath("cdp-sdx-medium-ha.bp"); + Blueprint bp = new Blueprint(); + bp.setBlueprintText(mediumDutyBP); + ReflectionTestUtils.setField(stack, "instanceGroups", instanceGroups); + ReflectionTestUtils.setField(stack, "blueprint", bp); + underTest.redeployGatewayPillarOnly(stack); + ArgumentCaptor saltConfigCaptor = ArgumentCaptor.forClass(SaltConfig.class); + verify(hostOrchestrator).uploadGatewayPillar(any(), allNodesCaptor.capture(), any(), saltConfigCaptor.capture()); + Set allNodes = allNodesCaptor.getValue(); + assertEquals(5, allNodes.size()); + SaltConfig saltConfig = saltConfigCaptor.getValue(); + SaltPillarProperties gatewayPillarProperties = saltConfig.getServicePillarConfig().get("gateway"); + Map props = (HashMap) gatewayPillarProperties.getProperties().get("gateway"); + Map> location = (HashMap>) props.get("location"); + List ranger = location.get("RANGER"); + assertTrue(ranger.contains("master1")); + } + + @Test + @MockitoSettings(strictness = Strictness.LENIENT) + void testLDRangerToplogy() throws IOException, CloudbreakOrchestratorFailedException { + setupMocksForRunClusterServices(); + Set nodes = Sets.newHashSet(node("fqdn1"), node("fqdn2"), node("fqdn3"), + node("gateway1"), node("master1")); + when(stackUtil.collectNodes(any())).thenReturn(nodes); + when(exposedServiceCollector.getAllServiceNames()).thenReturn(Set.of("RANGER")); + Map> locations = new HashMap<>(); + locations.put("RANGER", List.of("master1", "master2")); + when(componentLocator.getComponentLocation(any(), any())).thenReturn(locations); + StackView stackViewMock = mock(StackView.class); + when(stackViewMock.getId()).thenReturn(STACK_ID); + ReflectionTestUtils.setField(stack, "stack", stackViewMock); + InstanceGroupView instanceGroupView = mock(InstanceGroupView.class); + InstanceMetadataView instanceMetadataView = mock(InstanceMetadataView.class); + when(instanceMetadataView.getInstanceMetadataType()).thenReturn(InstanceMetadataType.GATEWAY_PRIMARY); + when(instanceGroupView.getGroupName()).thenReturn("GATEWAY"); + Map instanceGroups = new HashMap<>(); + InstanceGroupDto instanceGroupDto = new InstanceGroupDto(instanceGroupView, List.of(instanceMetadataView)); + instanceGroups.put("GATEWAY", instanceGroupDto); + String lightDutyBP = FileReaderUtils.readFileFromClasspath("cdp-sdx.bp"); + Blueprint bp = new Blueprint(); + bp.setBlueprintText(lightDutyBP); + ReflectionTestUtils.setField(stack, "instanceGroups", instanceGroups); + ReflectionTestUtils.setField(stack, "blueprint", bp); + underTest.redeployGatewayPillarOnly(stack); + ArgumentCaptor saltConfigCaptor = ArgumentCaptor.forClass(SaltConfig.class); + verify(hostOrchestrator).uploadGatewayPillar(any(), allNodesCaptor.capture(), any(), saltConfigCaptor.capture()); + Set allNodes = allNodesCaptor.getValue(); + assertEquals(5, allNodes.size()); + SaltConfig saltConfig = saltConfigCaptor.getValue(); + SaltPillarProperties gatewayPillarProperties = saltConfig.getServicePillarConfig().get("gateway"); + Map props = (HashMap) gatewayPillarProperties.getProperties().get("gateway"); + Map> location = (HashMap>) props.get("location"); + List ranger = location.get("RANGER"); + assertTrue(ranger.contains("master1")); + } + private void createInstanceGroup(Template template, List instanceGroups, String fqdn1, String fqdn2, String privateIp1, String privateIp2) { InstanceGroup instanceGroup = new InstanceGroup(); diff --git a/core/src/test/resources/cdp-sdx-enterprise.bp b/core/src/test/resources/cdp-sdx-enterprise.bp new file mode 100644 index 00000000000..e23a559196f --- /dev/null +++ b/core/src/test/resources/cdp-sdx-enterprise.bp @@ -0,0 +1,624 @@ +{ + "tags":{ + "shared_services_ready":true + }, + "description":"7.2.18 - Enterprise SDX template with Atlas, HMS, Ranger and other services they are dependent on. Services like HDFS, HBASE, RANGER, HMS have HA", + "blueprint":{ + "cdhVersion":"7.2.18", + "displayName":"enterprise-datalake", + "hostTemplates":[ + { + "cardinality":2, + "refName":"master", + "roleConfigGroupsRefNames":[ + "zookeeper-SERVER-BASE", + "hbase-MASTER-BASE", + "ranger-RANGER_ADMIN-BASE", + "hdfs-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE", + "solr-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hdfs-NAMENODE-BASE", + "hdfs-FAILOVERCONTROLLER-BASE", + "hdfs-JOURNALNODE-BASE" + ] + }, + { + "cardinality":2, + "refName":"gateway", + "roleConfigGroupsRefNames":[ + "atlas-ATLAS_SERVER-BASE", + "ranger-RANGER_USERSYNC-BASE", + "ranger-RANGER_TAGSYNC-BASE", + "hive-HIVEMETASTORE-BASE", + "ranger-RANGER_ADMIN-BASE", + "knox-KNOX_GATEWAY-BASE", + "solr-GATEWAY-BASE", + "hdfs-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE" + ] + }, + { + "cardinality":3, + "refName":"core", + "roleConfigGroupsRefNames":[ + "hdfs-DATANODE-BASE", + "kafka-KAFKA_BROKER-BASE", + "solr-SOLR_SERVER-BASE", + "hbase-REGIONSERVER-BASE", + "solr-GATEWAY-BASE", + "hdfs-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE" + ] + }, + { + "cardinality":1, + "refName":"auxiliary", + "roleConfigGroupsRefNames":[ + "zookeeper-SERVER-BASE", + "hdfs-JOURNALNODE-BASE" + ] + }, + { + "cardinality":2, + "refName":"idbroker", + "roleConfigGroupsRefNames":[ + "knox-IDBROKER-BASE" + ] + }, + { + "cardinality":0, + "refName":"solrHG", + "roleConfigGroupsRefNames":[ + "solr-SOLR_SERVER-EXTERNAL", + "solr-GATEWAY-BASE" + ] + }, + { + "cardinality":0, + "refName":"storageHG", + "roleConfigGroupsRefNames":[ + "hbase-REGIONSERVER-EXTERNAL", + "hdfs-DATANODE-EXTERNAL", + "hbase-GATEWAY-BASE", + "hdfs-GATEWAY-BASE" + ] + }, + { + "cardinality":0, + "refName":"kafkaHG", + "roleConfigGroupsRefNames":[ + "kafka-KAFKA_BROKER-EXTERNAL", + "kafka-GATEWAY-BASE" + ] + }, + { + "cardinality":0, + "refName":"razHG", + "roleConfigGroupsRefNames":[ + ] + }, + { + "cardinality":0, + "refName":"atlasHG", + "roleConfigGroupsRefNames":[ + "atlas-ATLAS_SERVER-EXTERNAL", + "hdfs-GATEWAY-BASE", + "hbase-GATEWAY-BASE" + ] + }, + { + "cardinality":0, + "refName":"hmsHG", + "roleConfigGroupsRefNames":[ + "hive-HIVEMETASTORE-EXTERNAL", + "hive-GATEWAY-BASE" + ] + } + ], + "services":[ + { + "refName":"zookeeper", + "roleConfigGroups":[ + { + "base":true, + "refName":"zookeeper-SERVER-BASE", + "roleType":"SERVER" + } + ], + "serviceType":"ZOOKEEPER", + "serviceConfigs":[ + { + "name":"service_config_suppression_server_count_validator", + "value":"true" + } + ] + }, + { + "refName":"ranger", + "roleConfigGroups":[ + { + "base":true, + "refName":"ranger-RANGER_USERSYNC-BASE", + "roleType":"RANGER_USERSYNC", + "configs":[ + { + "name":"role_config_suppression_ranger_usersync_max_heap_size", + "value":"true" + } + ] + }, + { + "base":true, + "refName":"ranger-RANGER_TAGSYNC-BASE", + "roleType":"RANGER_TAGSYNC", + "configs":[ + { + "name":"role_config_suppression_ranger_tagsync_max_heap_size", + "value":"true" + } + ] + }, + { + "base":true, + "refName":"ranger-RANGER_ADMIN-BASE", + "roleType":"RANGER_ADMIN", + "configs":[ + { + "name":"role_config_suppression_ranger_admin_max_heap_size", + "value":"true" + }, + { + "name":"ranger.audit.solr.no.replica", + "value":"2" + }, + { + "name":"ranger.audit.solr.no.shards", + "value":"3" + }, + { + "name":"ranger.audit.solr.max.shards.per.node", + "value":"2" + }, + { + "name":"ranger.audit.solr.config.ttl", + "value":"30" + } + ] + } + ], + "serviceConfigs":[ + { + "name":"hdfs_service", + "ref":"hdfs" + }, + { + "name":"rangeradmin_user_password", + "value":"{{{ general.password }}}" + }, + { + "name":"rangertagsync_user_password", + "value":"{{{ general.password }}}" + }, + { + "name":"solr_service", + "ref":"solr" + }, + { + "name":"rangerusersync_user_password", + "value":"{{{ general.password }}}" + }, + { + "name":"keyadmin_user_password", + "value":"{{{ general.password }}}" + } + ], + "serviceType":"RANGER" + }, + { + "refName":"atlas", + "roleConfigGroups":[ + { + "base":true, + "configs":[ + { + "name":"role_config_suppression_atlas_max_heap_size", + "value":"true" + }, + { + "name":"atlas_authentication_method_file", + "value":"false" + }, + { + "name":"atlas_solr_shards", + "value":"3" + }, + { + "name":"atlas_solr_replication_factor", + "value":"2" + }, + { + "name":"atlas_ssl_exclude_protocols", + "value":"TLSv1,TLSv1.1" + }, + { + "name":"atlas_max_heap_size", + "value":8192 + } + ], + "refName":"atlas-ATLAS_SERVER-BASE", + "roleType":"ATLAS_SERVER" + }, + { + "base":false, + "configs":[ + { + "name":"atlas_authentication_method_file", + "value":"false" + }, + { + "name":"atlas_solr_shards", + "value":"3" + }, + { + "name":"atlas_solr_replication_factor", + "value":"2" + }, + { + "name":"atlas_ssl_exclude_protocols", + "value":"TLSv1,TLSv1.1" + }, + { + "name":"atlas_max_heap_size", + "value":11776 + } + ], + "refName":"atlas-ATLAS_SERVER-EXTERNAL", + "roleType":"ATLAS_SERVER" + } + ], + "serviceConfigs":[ + { + "name":"solr_service", + "ref":"solr" + }, + { + "name":"kafka_service", + "ref":"kafka" + }, + { + "name":"hbase_service", + "ref":"hbase" + }, + { + "name":"hdfs_service", + "ref":"hdfs" + } + ], + "serviceType":"ATLAS" + }, + { + "refName":"solr", + "roleConfigGroups":[ + { + "base":true, + "configs":[ + { + "name":"solr_https_port", + "value":"8985" + }, + { + "name":"solr_java_opts", + "value":"{{JAVA_GC_ARGS}} -Dsolr.hdfs.allow.location.override=true" + }, + { + "name":"solr_java_heapsize", + "value":8589934592 + }, + { + "name":"solr_java_direct_memory_size", + "value":1073741824 + } + ], + "refName":"solr-SOLR_SERVER-BASE", + "roleType":"SOLR_SERVER" + }, + { + "base":false, + "configs":[ + { + "name":"solr_https_port", + "value":"8985" + }, + { + "name":"solr_java_opts", + "value":"{{JAVA_GC_ARGS}} -Dsolr.hdfs.allow.location.override=true" + }, + { + "name":"solr_java_heapsize", + "value":12348030976 + }, + { + "name":"solr_java_direct_memory_size", + "value":1073741824 + } + ], + "refName":"solr-SOLR_SERVER-EXTERNAL", + "roleType":"SOLR_SERVER" + }, + { + "base":true, + "refName":"solr-GATEWAY-BASE", + "roleType":"GATEWAY" + } + ], + "serviceConfigs":[ + { + "name":"hdfs_service", + "ref":"hdfs" + }, + { + "name":"zookeeper_service", + "ref":"zookeeper" + }, + { + "name":"enable_ranger_authorization", + "value":"true" + } + ], + "serviceType":"SOLR" + }, + { + "refName":"hbase", + "roleConfigGroups":[ + { + "base":true, + "refName":"hbase-GATEWAY-BASE", + "roleType":"GATEWAY" + }, + { + "base":true, + "refName":"hbase-REGIONSERVER-BASE", + "roleType":"REGIONSERVER", + "configs":[ + { + "name":"hbase_bucketcache_ioengine", + "value":"" + }, + { + "name":"hbase_regionserver_java_heapsize", + "value":8589934592 + } + ] + }, + { + "base":false, + "refName":"hbase-REGIONSERVER-EXTERNAL", + "roleType":"REGIONSERVER", + "configs":[ + { + "name":"hbase_bucketcache_ioengine", + "value":"" + }, + { + "name":"hbase_regionserver_java_heapsize", + "value":8589934592 + } + ] + }, + { + "base":true, + "refName":"hbase-MASTER-BASE", + "roleType":"MASTER" + }, + { + "base":true, + "refName":"hbase-HBASETHRIFTSERVER-BASE", + "roleType":"HBASETHRIFTSERVER" + } + ], + "serviceConfigs":[ + { + "name":"hbase_wal_dir", + "value":"/hbase-wals" + }, + { + "name":"service_config_suppression_hadoop_secure_web_ui", + "value":"true" + } + ], + "serviceType":"HBASE" + }, + { + "refName":"hdfs", + "roleConfigGroups":[ + { + "base":true, + "refName":"hdfs-NAMENODE-BASE", + "roleType":"NAMENODE", + "configs":[ + { + "name":"role_config_suppression_namenode_java_heapsize_minimum_validator", + "value":"true" + }, + { + "name":"namenode_java_heapsize", + "value":1073741824 + }, + { + "name":"datanode_java_heapsize", + "value":3219128320 + } + ] + }, + { + "base":true, + "refName":"hdfs-SECONDARYNAMENODE-BASE", + "roleType":"SECONDARYNAMENODE" + }, + { + "base":true, + "refName":"hdfs-DATANODE-BASE", + "roleType":"DATANODE", + "configs":[ + { + "name":"dfs_datanode_max_locked_memory", + "value":0 + } + ] + }, + { + "base":false, + "refName":"hdfs-DATANODE-EXTERNAL", + "roleType":"DATANODE", + "configs":[ + { + "name":"dfs_datanode_max_locked_memory", + "value":0 + } + ] + }, + { + "base":true, + "refName":"hdfs-BALANCER-BASE", + "roleType":"BALANCER" + }, + { + "base":true, + "refName":"hdfs-GATEWAY-BASE", + "roleType":"GATEWAY" + }, + { + "base":true, + "refName":"hdfs-FAILOVERCONTROLLER-BASE", + "roleType":"FAILOVERCONTROLLER" + }, + { + "base":true, + "refName":"hdfs-JOURNALNODE-BASE", + "roleType":"JOURNALNODE" + } + ], + "serviceConfigs":[ + { + "name":"enable_ranger_authorization", + "value":"true" + }, + { + "name":"service_health_suppression_hdfs_verify_ec_with_topology", + "value":"true" + }, + { + "name":"dfs_replication", + "value":"2" + }, + { + "name":"service_config_suppression_datanode_count_validator", + "value":"true" + } + ], + "serviceType":"HDFS" + }, + { + "refName":"kafka", + "roleConfigGroups":[ + { + "base":true, + "refName":"kafka-GATEWAY-BASE", + "roleType":"GATEWAY" + }, + { + "base":true, + "refName":"kafka-KAFKA_BROKER-BASE", + "roleType":"KAFKA_BROKER" + }, + { + "base":false, + "refName":"kafka-KAFKA_BROKER-EXTERNAL", + "roleType":"KAFKA_BROKER", + "configs": [ + { + "name": "broker_max_heap_size", + "value": 3072 + }, + { + "name": "ssl_enabled", + "value": "true" + } + ] + } + ], + "serviceConfigs":[ + { + "name":"zookeeper_service", + "ref":"zookeeper" + }, + { + "name":"default.replication.factor", + "value":"3" + }, + { + "name":"min.insync.replicas", + "value":"2" + } + ], + "serviceType":"KAFKA" + }, + { + "serviceType":"KNOX", + "refName":"knox", + "roleConfigGroups":[ + { + "base":true, + "refName":"knox-KNOX_GATEWAY-BASE", + "roleType":"KNOX_GATEWAY" + }, + { + "base":true, + "refName":"knox-IDBROKER-BASE", + "roleType":"IDBROKER" + } + ] + }, + { + "refName":"hive", + "displayName":"Hive Metastore", + "roleConfigGroups":[ + { + "base":true, + "refName":"hive-GATEWAY-BASE", + "roleType":"GATEWAY" + }, + { + "base":true, + "refName":"hive-HIVEMETASTORE-BASE", + "roleType":"HIVEMETASTORE", + "configs":[ + { + "name":"hive_metastore_java_heapsize", + "value":8589934592 + } + ] + }, + { + "base":false, + "refName":"hive-HIVEMETASTORE-EXTERNAL", + "roleType":"HIVEMETASTORE", + "configs":[ + { + "name":"hive_metastore_java_heapsize", + "value":12348030976 + } + ] + } + ], + "serviceType":"HIVE" + } + ] + } +} \ No newline at end of file diff --git a/core/src/test/resources/cdp-sdx-medium-ha.bp b/core/src/test/resources/cdp-sdx-medium-ha.bp new file mode 100644 index 00000000000..62ce15ec3a7 --- /dev/null +++ b/core/src/test/resources/cdp-sdx-medium-ha.bp @@ -0,0 +1,432 @@ +{ + "tags": { + "shared_services_ready": true + }, + "description": "7.2.18 - Medium SDX template with Atlas, HMS, Ranger and other services they are dependent on. Services like HDFS, HBASE, RANGER, HMS have HA", + "blueprint": { + "cdhVersion": "7.2.18", + "displayName": "medium-ha-datalake", + "hostTemplates": [ + { + "cardinality": 2, + "refName": "master", + "roleConfigGroupsRefNames": [ + "hive-HIVEMETASTORE-BASE", + "ranger-RANGER_ADMIN-BASE", + "hdfs-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE", + "solr-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hdfs-NAMENODE-BASE", + "hdfs-FAILOVERCONTROLLER-BASE" + ] + }, + { + "cardinality": 2, + "refName": "gateway", + "roleConfigGroupsRefNames": [ + "atlas-ATLAS_SERVER-BASE", + "ranger-RANGER_ADMIN-BASE", + "zookeeper-SERVER-BASE", + "knox-KNOX_GATEWAY-BASE", + "solr-GATEWAY-BASE", + "hdfs-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE" + ] + }, + { + "cardinality": 3, + "refName": "core", + "roleConfigGroupsRefNames": [ + "hdfs-DATANODE-BASE", + "hdfs-JOURNALNODE-BASE", + "kafka-KAFKA_BROKER-BASE", + "solr-SOLR_SERVER-BASE", + "hbase-REGIONSERVER-BASE", + "hbase-MASTER-BASE", + "solr-GATEWAY-BASE", + "hdfs-GATEWAY-BASE", + "kafka-GATEWAY-BASE", + "hive-GATEWAY-BASE", + "hbase-GATEWAY-BASE" + ] + }, + { + "cardinality": 1, + "refName": "auxiliary", + "roleConfigGroupsRefNames": [ + "ranger-RANGER_USERSYNC-BASE", + "ranger-RANGER_TAGSYNC-BASE", + "zookeeper-SERVER-BASE" + ] + }, + { + "cardinality": 2, + "refName": "idbroker", + "roleConfigGroupsRefNames": [ + "knox-IDBROKER-BASE" + ] + } + ], + "services": [ + { + "refName": "zookeeper", + "roleConfigGroups": [ + { + "base": true, + "refName": "zookeeper-SERVER-BASE", + "roleType": "SERVER" + } + ], + "serviceType": "ZOOKEEPER", + "serviceConfigs": [ + { + "name": "service_config_suppression_server_count_validator", + "value": "true" + } + ] + }, + { + "refName": "ranger", + "roleConfigGroups": [ + { + "base": true, + "refName": "ranger-RANGER_USERSYNC-BASE", + "roleType": "RANGER_USERSYNC", + "configs": [ + { + "name": "role_config_suppression_ranger_usersync_max_heap_size", + "value": "true" + } + ] + }, + { + "base": true, + "refName": "ranger-RANGER_TAGSYNC-BASE", + "roleType": "RANGER_TAGSYNC", + "configs": [ + { + "name": "role_config_suppression_ranger_tagsync_max_heap_size", + "value": "true" + } + ] + }, + { + "base": true, + "refName": "ranger-RANGER_ADMIN-BASE", + "roleType": "RANGER_ADMIN", + "configs": [ + { + "name": "role_config_suppression_ranger_admin_max_heap_size", + "value": "true" + }, + { + "name": "ranger.audit.solr.no.replica", + "value": "2" + }, + { + "name": "ranger.audit.solr.no.shards", + "value": "3" + }, + { + "name": "ranger.audit.solr.max.shards.per.node", + "value": "2" + } + ] + } + ], + "serviceConfigs": [ + { + "name": "hdfs_service", + "ref": "hdfs" + }, + { + "name": "rangeradmin_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "rangertagsync_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "solr_service", + "ref": "solr" + }, + { + "name": "rangerusersync_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "keyadmin_user_password", + "value": "{{{ general.password }}}" + } + ], + "serviceType": "RANGER" + }, + { + "refName": "atlas", + "roleConfigGroups": [ + { + "base": true, + "configs": [ + { + "name": "role_config_suppression_atlas_max_heap_size", + "value": "true" + }, + { + "name": "atlas_authentication_method_file", + "value": "false" + }, + { + "name": "atlas_solr_shards", + "value": "3" + }, + { + "name": "atlas_solr_replication_factor", + "value": "2" + }, + { + "name": "atlas_ssl_exclude_protocols", + "value": "TLSv1,TLSv1.1" + } + ], + "refName": "atlas-ATLAS_SERVER-BASE", + "roleType": "ATLAS_SERVER" + } + ], + "serviceConfigs": [ + { + "name": "solr_service", + "ref": "solr" + }, + { + "name": "kafka_service", + "ref": "kafka" + }, + { + "name": "hbase_service", + "ref": "hbase" + }, + { + "name": "hdfs_service", + "ref": "hdfs" + } + ], + "serviceType": "ATLAS" + }, + { + "refName": "solr", + "roleConfigGroups": [ + { + "base": true, + "configs": [ + { + "name": "solr_https_port", + "value": "8985" + }, + { + "name": "solr_java_opts", + "value": "{{JAVA_GC_ARGS}} -Dsolr.hdfs.allow.location.override=true" + } + ], + "refName": "solr-SOLR_SERVER-BASE", + "roleType": "SOLR_SERVER" + }, + { + "base": true, + "refName": "solr-GATEWAY-BASE", + "roleType": "GATEWAY" + } + ], + "serviceConfigs": [ + { + "name": "hdfs_service", + "ref": "hdfs" + }, + { + "name": "zookeeper_service", + "ref": "zookeeper" + }, + { + "name": "enable_ranger_authorization", + "value": "true" + } + ], + "serviceType": "SOLR" + }, + { + "refName": "hbase", + "roleConfigGroups": [ + { + "base": true, + "refName": "hbase-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "hbase-REGIONSERVER-BASE", + "roleType": "REGIONSERVER", + "configs": [ + { + "name": "hbase_bucketcache_ioengine", + "value": "" + } + ] + }, + { + "base": true, + "refName": "hbase-MASTER-BASE", + "roleType": "MASTER" + }, + { + "base": true, + "refName": "hbase-HBASETHRIFTSERVER-BASE", + "roleType": "HBASETHRIFTSERVER" + } + ], + "serviceConfigs": [ + { + "name": "hbase_wal_dir", + "value": "/hbase-wals" + }, + { + "name": "service_config_suppression_hadoop_secure_web_ui", + "value": "true" + } + ], + "serviceType": "HBASE" + }, + { + "refName": "hdfs", + "roleConfigGroups": [ + { + "base": true, + "refName": "hdfs-NAMENODE-BASE", + "roleType": "NAMENODE", + "configs": [ + { + "name": "role_config_suppression_namenode_java_heapsize_minimum_validator", + "value": "true" + } + ] + }, + { + "base": true, + "refName": "hdfs-SECONDARYNAMENODE-BASE", + "roleType": "SECONDARYNAMENODE" + }, + { + "base": true, + "refName": "hdfs-DATANODE-BASE", + "roleType": "DATANODE" + }, + { + "base": true, + "refName": "hdfs-BALANCER-BASE", + "roleType": "BALANCER" + }, + { + "base": true, + "refName": "hdfs-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "hdfs-FAILOVERCONTROLLER-BASE", + "roleType": "FAILOVERCONTROLLER" + }, + { + "base": true, + "refName": "hdfs-JOURNALNODE-BASE", + "roleType": "JOURNALNODE" + } + ], + "serviceConfigs": [ + { + "name": "enable_ranger_authorization", + "value": "true" + }, + { + "name": "service_health_suppression_hdfs_verify_ec_with_topology", + "value": "true" + }, + { + "name": "dfs_replication", + "value": "2" + }, + { + "name": "service_config_suppression_datanode_count_validator", + "value": "true" + } + ], + "serviceType": "HDFS" + }, + { + "refName": "kafka", + "roleConfigGroups": [ + { + "base": true, + "refName": "kafka-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "kafka-KAFKA_BROKER-BASE", + "roleType": "KAFKA_BROKER" + } + ], + "serviceConfigs": [ + { + "name": "zookeeper_service", + "ref": "zookeeper" + }, + { + "name": "default.replication.factor", + "value": "3" + }, + { + "name": "min.insync.replicas", + "value": "2" + } + ], + "serviceType": "KAFKA" + }, + { + "serviceType": "KNOX", + "refName": "knox", + "roleConfigGroups": [ + { + "base": true, + "refName": "knox-KNOX_GATEWAY-BASE", + "roleType": "KNOX_GATEWAY" + }, + { + "base": true, + "refName": "knox-IDBROKER-BASE", + "roleType": "IDBROKER" + } + ] + }, + { + "refName": "hive", + "displayName": "Hive Metastore", + "roleConfigGroups": [ + { + "base": true, + "refName": "hive-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "hive-HIVEMETASTORE-BASE", + "roleType": "HIVEMETASTORE" + } + ], + "serviceType": "HIVE" + } + ] + } +} diff --git a/core/src/test/resources/cdp-sdx.bp b/core/src/test/resources/cdp-sdx.bp new file mode 100644 index 00000000000..fd3a3475d7a --- /dev/null +++ b/core/src/test/resources/cdp-sdx.bp @@ -0,0 +1,377 @@ +{ + "tags": { + "shared_services_ready": true + }, + "description": "7.2.18 - SDX template with Atlas, HMS, Ranger and other services they are dependent on", + "blueprint": { + "cdhVersion": "7.2.18", + "displayName": "datalake", + "hostTemplates": [ + { + "cardinality": 1, + "refName": "master", + "roleConfigGroupsRefNames": [ + "atlas-ATLAS_SERVER-BASE", + "hbase-GATEWAY-BASE", + "hbase-HBASETHRIFTSERVER-BASE", + "hbase-REGIONSERVER-BASE", + "hbase-MASTER-BASE", + "hdfs-DATANODE-BASE", + "hdfs-BALANCER-BASE", + "hdfs-GATEWAY-BASE", + "hdfs-NAMENODE-BASE", + "hdfs-SECONDARYNAMENODE-BASE", + "hive-HIVEMETASTORE-BASE", + "kafka-GATEWAY-BASE", + "kafka-KAFKA_BROKER-BASE", + "knox-KNOX_GATEWAY-BASE", + "ranger-RANGER_ADMIN-BASE", + "ranger-RANGER_TAGSYNC-BASE", + "ranger-RANGER_USERSYNC-BASE", + "solr-SOLR_SERVER-BASE", + "zookeeper-SERVER-BASE" + ] + }, + { + "cardinality": 1, + "refName": "idbroker", + "roleConfigGroupsRefNames": [ + "knox-IDBROKER-BASE" + ] + } + ], + "services": [ + { + "refName": "zookeeper", + "roleConfigGroups": [ + { + "base": true, + "refName": "zookeeper-SERVER-BASE", + "roleType": "SERVER" + } + ], + "serviceType": "ZOOKEEPER", + "serviceConfigs": [ + { + "name": "service_config_suppression_server_count_validator", + "value": "true" + } + ] + }, + { + "refName": "ranger", + "roleConfigGroups": [ + { + "base": true, + "refName": "ranger-RANGER_USERSYNC-BASE", + "roleType": "RANGER_USERSYNC", + "configs": [ + { + "name" : "role_config_suppression_ranger_usersync_max_heap_size", + "value" : "true" + } + ] + }, + { + "base": true, + "refName": "ranger-RANGER_TAGSYNC-BASE", + "roleType": "RANGER_TAGSYNC", + "configs": [ + { + "name": "role_config_suppression_ranger_tagsync_max_heap_size", + "value": "true" + } + ] + }, + { + "base": true, + "refName": "ranger-RANGER_ADMIN-BASE", + "roleType": "RANGER_ADMIN", + "configs": [ + { + "name": "role_config_suppression_ranger_admin_max_heap_size", + "value": "true" + } + ] + } + ], + "serviceConfigs": [ + { + "name": "hdfs_service", + "ref": "hdfs" + }, + { + "name": "rangeradmin_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "rangertagsync_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "solr_service", + "ref": "solr" + }, + { + "name": "rangerusersync_user_password", + "value": "{{{ general.password }}}" + }, + { + "name": "keyadmin_user_password", + "value": "{{{ general.password }}}" + } + ], + "serviceType": "RANGER" + }, + { + "refName": "atlas", + "roleConfigGroups": [ + { + "base": true, + "configs": [ + { + "name": "role_config_suppression_atlas_max_heap_size", + "value": "true" + }, + { + "name": "atlas_authentication_method_file", + "value": "false" + }, + { + "name": "atlas_ssl_exclude_protocols", + "value": "TLSv1,TLSv1.1" + } + ], + "refName": "atlas-ATLAS_SERVER-BASE", + "roleType": "ATLAS_SERVER" + } + ], + "serviceConfigs": [ + { + "name": "solr_service", + "ref": "solr" + }, + { + "name": "kafka_service", + "ref": "kafka" + }, + { + "name": "hbase_service", + "ref": "hbase" + }, + { + "name": "hdfs_service", + "ref": "hdfs" + } + ], + "serviceType": "ATLAS" + }, + { + "refName": "solr", + "roleConfigGroups": [ + { + "base": true, + "configs": [ + { + "name": "solr_https_port", + "value": "8985" + }, + { + "name": "solr_java_opts", + "value": "{{JAVA_GC_ARGS}} -Dsolr.hdfs.allow.location.override=true" + } + ], + "refName": "solr-SOLR_SERVER-BASE", + "roleType": "SOLR_SERVER" + } + ], + "serviceConfigs": [ + { + "name": "hdfs_service", + "ref": "hdfs" + }, + { + "name": "zookeeper_service", + "ref": "zookeeper" + }, + { + "name": "enable_ranger_authorization", + "value": "true" + } + ], + "serviceType": "SOLR" + }, + { + "refName": "hbase", + "roleConfigGroups": [ + { + "base": true, + "refName": "hbase-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "hbase-REGIONSERVER-BASE", + "roleType": "REGIONSERVER", + "configs": [ + { + "name": "hbase_bucketcache_ioengine", + "value": "" + } + ] + }, + { + "base": true, + "refName": "hbase-MASTER-BASE", + "roleType": "MASTER" + }, + { + "base": true, + "refName": "hbase-HBASETHRIFTSERVER-BASE", + "roleType": "HBASETHRIFTSERVER" + } + ], + "serviceConfigs": [ + { + "name": "hbase_wal_dir", + "value": "/hbase-wals" + }, + { + "name": "service_config_suppression_hadoop_secure_web_ui", + "value": "true" + } + ], + "serviceType": "HBASE" + }, + { + "refName": "hdfs", + "roleConfigGroups": [ + { + "base": true, + "refName": "hdfs-NAMENODE-BASE", + "roleType": "NAMENODE", + "configs": [ + { + "name": "role_config_suppression_namenode_java_heapsize_minimum_validator", + "value": "true" + } + ] + }, + { + "base": true, + "refName": "hdfs-SECONDARYNAMENODE-BASE", + "roleType": "SECONDARYNAMENODE" + }, + { + "base": true, + "refName": "hdfs-DATANODE-BASE", + "roleType": "DATANODE" + }, + { + "base": true, + "refName": "hdfs-BALANCER-BASE", + "roleType": "BALANCER" + }, + { + "base": true, + "refName": "hdfs-GATEWAY-BASE", + "roleType": "GATEWAY" + } + ], + "serviceConfigs": [ + { + "name": "enable_ranger_authorization", + "value": "true" + }, + { + "name": "service_health_suppression_hdfs_verify_ec_with_topology", + "value": "true" + }, + { + "name": "dfs_replication", + "value": "1" + }, + { + "name": "service_config_suppression_datanode_count_validator", + "value": "true" + } + ], + "serviceType": "HDFS" + }, + { + "refName": "kafka", + "roleConfigGroups": [ + { + "base": true, + "refName": "kafka-GATEWAY-BASE", + "roleType": "GATEWAY" + }, + { + "base": true, + "refName": "kafka-KAFKA_BROKER-BASE", + "roleType": "KAFKA_BROKER" + } + ], + "serviceConfigs": [ + { + "name": "zookeeper_service", + "ref": "zookeeper" + }, + { + "name": "offsets.topic.replication.factor", + "value": "1" + }, + { + "name": "transaction.state.log.replication.factor", + "value": "1" + }, + { + "name": "transaction.state.log.min.isr", + "value": "1" + }, + { + "name": "service_config_suppression_offsets.topic.replication.factor", + "value": "true" + }, + { + "name": "service_config_suppression_transaction.state.log.replication.factor", + "value": "true" + }, + { + "name": "service_config_suppression_kafka_broker_count_validator", + "value": "true" + } + ], + "serviceType": "KAFKA" + }, + { + "serviceType": "KNOX", + "refName": "knox", + "roleConfigGroups": [ + { + "base": true, + "refName": "knox-KNOX_GATEWAY-BASE", + "roleType": "KNOX_GATEWAY" + }, + { + "base": true, + "refName": "knox-IDBROKER-BASE", + "roleType": "IDBROKER" + } + ] + }, + { + "refName": "hive", + "displayName": "Hive Metastore", + "roleConfigGroups": [ + { + "base": true, + "refName": "hive-HIVEMETASTORE-BASE", + "roleType": "HIVEMETASTORE" + } + ], + "serviceType": "HIVE" + } + ] + } +}