Add safe-guard to omit void partitions

Netflix · Sep 9, 2024 · 6bc27b5 · 6bc27b5
1 parent dccc193
commit 6bc27b5
Show file tree

Hide file tree

Showing 13 changed files with 101 additions and 14 deletions.
diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java
@@ -642,5 +642,12 @@ public interface Config {
      * @return parentChildRelationshipProperties
      */
     ParentChildRelationshipProperties getParentChildRelationshipProperties();
+
+    /**
+     * Whether we want to omit void transform partition from current partitions.
+     *
+     * @return True if it should be omitted
+     */
+    boolean omitVoidTransformEnabled();
 }
 
diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java
@@ -703,4 +703,9 @@ public boolean isParentChildDropEnabled() {
     public ParentChildRelationshipProperties getParentChildRelationshipProperties() {
         return this.metacatProperties.getParentChildRelationshipProperties();
     }
+
+    @Override
+    public boolean omitVoidTransformEnabled() {
+        return this.metacatProperties.getPartitionColumnProperties().isOmitVoidPartitionEnabled();
+    }
 }
diff --git a/...-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java b/...-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java
@@ -72,6 +72,8 @@ public class MetacatProperties {
     private RateLimiterProperties rateLimiterProperties = new RateLimiterProperties();
     @NonNull
     private ParentChildRelationshipProperties parentChildRelationshipProperties;
+    @NonNull
+    private PartitionColumnProperties partitionColumnProperties;
 
     /**
      * Constructor for MetacatProperties.
@@ -81,5 +83,6 @@ public class MetacatProperties {
     public MetacatProperties(final Environment env) {
         this.env = env;
         this.parentChildRelationshipProperties = new ParentChildRelationshipProperties(env);
+        this.partitionColumnProperties = new PartitionColumnProperties(env);
     }
 }
diff --git a/...src/main/java/com/netflix/metacat/common/server/properties/PartitionColumnProperties.java b/...src/main/java/com/netflix/metacat/common/server/properties/PartitionColumnProperties.java
@@ -0,0 +1,43 @@
+package com.netflix.metacat.common.server.properties;
+
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.core.env.Environment;
+import javax.annotation.Nullable;
+
+/**
+ * Partition Column service properties.
+ *
+ * @author gabeiglio
+ */
+@Data
+@Slf4j
+public class PartitionColumnProperties {
+    private static final String OMIT_VOID_TRANSFORM_PARTITION_FIELDS_PROPERTY_NAME =
+        "metacat.partitionColumnProperties.omitVoidTransformPartitionFields";
+    private boolean omitVoidPartitionEnabled;
+
+    /**
+     * Constructor.
+     *
+     * @param env Spring environment
+     */
+    public PartitionColumnProperties(@Nullable final Environment env) {
+        if (env != null) {
+            setOmitVoidTransformPartitionFields(
+                env.getProperty(OMIT_VOID_TRANSFORM_PARTITION_FIELDS_PROPERTY_NAME,
+                    Boolean.class, true)
+            );
+        }
+    }
+
+    /**
+     * set omitVoidPartitionEnabled based on boolean config.
+     *
+     * @param  configBool configBool
+     */
+    public void setOmitVoidTransformPartitionFields(final boolean configBool) {
+        this.omitVoidPartitionEnabled = configBool;
+    }
+
+}
diff --git a/...-connector-hive/src/main/java/com/netflix/metacat/connector/hive/HiveConnectorPlugin.java b/...-connector-hive/src/main/java/com/netflix/metacat/connector/hive/HiveConnectorPlugin.java
@@ -32,7 +32,8 @@
  */
 public class HiveConnectorPlugin implements ConnectorPlugin {
     private static final String CONNECTOR_TYPE = "hive";
-    private static final HiveTypeConverter HIVE_TYPE_CONVERTER = new HiveTypeConverter();
+    private static final HiveTypeConverter HIVE_TYPE_CONVERTER =
+        new HiveTypeConverter(ConnectorContext.builder().build().getConfig());
     private static final HiveConnectorInfoConverter INFO_CONVERTER_HIVE
         = new HiveConnectorInfoConverter(HIVE_TYPE_CONVERTER);
 

diff --git a/...c/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java b/...c/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java
@@ -65,7 +65,7 @@ public class HiveConnectorInfoConverter implements ConnectorInfoConverter<Databa
 
     private static final Splitter SLASH_SPLITTER = Splitter.on('/');
     private static final Splitter EQUAL_SPLITTER = Splitter.on('=').limit(2);
-    private HiveTypeConverter hiveTypeConverter = new HiveTypeConverter();
+    private final HiveTypeConverter hiveTypeConverter;
 
     /**
      * Constructor.

diff --git a/...r-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveTypeConverter.java b/...r-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveTypeConverter.java
@@ -17,6 +17,7 @@
 import com.google.common.collect.Lists;
 import com.netflix.metacat.common.server.connectors.ConnectorTypeConverter;
 import com.netflix.metacat.common.server.connectors.model.FieldInfo;
+import com.netflix.metacat.common.server.properties.Config;
 import com.netflix.metacat.common.type.BaseType;
 import com.netflix.metacat.common.type.CharType;
 import com.netflix.metacat.common.type.DecimalType;
@@ -74,6 +75,17 @@ public class HiveTypeConverter implements ConnectorTypeConverter {
     private static final Pattern DECIMAL_TYPE
         = Pattern.compile(DECIMAL_WITH_SCALE + "|" + DECIMAL_WITH_SCALE_AND_PRECISION, Pattern.CASE_INSENSITIVE);
 
+    private final Config config;
+
+    /**
+     * Constructor.
+     *
+     * @param config Environment config
+     */
+    public HiveTypeConverter(final Config config) {
+        this.config = config;
+    }
+
     private static Type getPrimitiveType(final TypeInfo typeInfo) {
         final PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
         if (HiveTypeMapping.getHIVE_TO_CANONICAL().containsKey(primitiveCategory.name())) {
@@ -111,11 +123,18 @@ public Type toMetacatType(final String type) {
     public List<FieldInfo> icebergSchemaTofieldDtos(final Schema schema,
                                                      final List<PartitionField> partitionFields) {
         final List<FieldInfo> fields = Lists.newArrayList();
-        final List<String> partitionNames =
-            partitionFields.stream()
+        final List<String> partitionNames;
+
+        if (config.omitVoidTransformEnabled()) {
+            partitionNames = partitionFields.stream()
                 .filter(f -> f.transform() != null && !f.transform().toString().equals("void"))
                 .map(f -> schema.findField(f.sourceId()).name())
                 .collect(Collectors.toList());
+        } else {
+            partitionNames = partitionFields.stream()
+                .map(f -> schema.findField(f.sourceId()).name())
+                .collect(Collectors.toList());
+        }
 
         for (Types.NestedField field : schema.columns()) {
             final FieldInfo fieldInfo = new FieldInfo();

diff --git a/...e/src/main/java/com/netflix/metacat/connector/hive/sql/HiveConnectorFastTableService.java b/...e/src/main/java/com/netflix/metacat/connector/hive/sql/HiveConnectorFastTableService.java
@@ -154,7 +154,8 @@ public TableInfo get(final ConnectorRequestContext requestContext, final Qualifi
                 && HiveTableUtil.isCommonView(info)) {
                 final String tableLoc = HiveTableUtil.getCommonViewMetadataLocation(info);
                 return hiveConnectorFastTableServiceProxy.getCommonViewTableInfo(name, tableLoc, info,
-                    new HiveTypeConverter(), connectorContext.getConfig().isIcebergCacheEnabled());
+                    new HiveTypeConverter(connectorContext.getConfig()),
+                        connectorContext.getConfig().isIcebergCacheEnabled());
             }
             if (!connectorContext.getConfig().isIcebergEnabled() || !HiveTableUtil.isIcebergTable(info)) {
                 return info;

diff --git a/...roovy/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConvertorSpec.groovy b/...roovy/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConvertorSpec.groovy
@@ -59,7 +59,8 @@ class HiveConnectorInfoConvertorSpec extends Specification{
     def setup() {
         // Stub this to always return true
         config.isEpochInSeconds() >> true
-        converter = new HiveConnectorInfoConverter( new HiveTypeConverter())
+        config.omitVoidTransformEnabled() >> true
+        converter = new HiveConnectorInfoConverter(new HiveTypeConverter(config))
     }
 
     def 'test date to epoch seconds'() {

diff --git a/...rc/test/groovy/com/netflix/metacat/connector/hive/converters/HiveTypeConverterSpec.groovy b/...rc/test/groovy/com/netflix/metacat/connector/hive/converters/HiveTypeConverterSpec.groovy
@@ -13,6 +13,7 @@
 
 package com.netflix.metacat.connector.hive.converters
 
+import com.netflix.metacat.common.server.properties.Config
 import org.apache.iceberg.PartitionField
 import org.apache.iceberg.transforms.Identity
 import org.apache.iceberg.transforms.VoidTransform
@@ -28,8 +29,14 @@ import spock.lang.Unroll
  * @since 1.0.0
  */
 class HiveTypeConverterSpec extends Specification {
-    @Shared
-    HiveTypeConverter converter = new HiveTypeConverter()
+    Config config = Mock(Config)
+    HiveTypeConverter converter;
+
+    def setup() {
+        // Stub omitVoidPartitionEnabled to always return true
+        this.config.omitVoidTransformEnabled() >> true
+        this.converter = new HiveTypeConverter(this.config)
+    }
 
     @Unroll
     def 'can convert "#typeString" to a presto type and back'(String typeString) {
@@ -276,10 +283,8 @@ class HiveTypeConverterSpec extends Specification {
             new PartitionField(2, 2, "field2", new VoidTransform<String>()),
         ]
 
-        def hiveTypeConverter = new HiveTypeConverter()
-
         when:
-        def fieldDtos = hiveTypeConverter.icebergSchemaTofieldDtos(initialSchema, initialPartitionFields)
+        def fieldDtos = this.converter.icebergSchemaTofieldDtos(initialSchema, initialPartitionFields)
 
         then:
         fieldDtos.size() == 3

diff --git a/...r-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorPlugin.java b/...r-polaris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorPlugin.java
@@ -16,7 +16,8 @@
 public class PolarisConnectorPlugin implements ConnectorPlugin {
 
     private static final String CONNECTOR_TYPE = "polaris";
-    private static final HiveTypeConverter TYPE_CONVERTER = new HiveTypeConverter();
+    private static final HiveTypeConverter TYPE_CONVERTER =
+        new HiveTypeConverter(ConnectorContext.builder().build().getConfig());
     private static final HiveConnectorInfoConverter INFO_CONVERTER
         = new HiveConnectorInfoConverter(TYPE_CONVERTER);
 

diff --git a/...src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java b/...src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java
@@ -93,7 +93,7 @@ public void init() {
             polarisStoreService,
             CATALOG_NAME,
             polarisDBService,
-            new HiveConnectorInfoConverter(new HiveTypeConverter()),
+            new HiveConnectorInfoConverter(new HiveTypeConverter(connectorContext.getConfig())),
             new IcebergTableHandler(connectorContext,
                 new IcebergTableCriteriaImpl(connectorContext),
                 new IcebergTableOpWrapper(connectorContext, serviceManager),

diff --git a/metacat-functional-tests/metacat-test-cluster/docker-compose.yml b/metacat-functional-tests/metacat-test-cluster/docker-compose.yml
@@ -77,7 +77,8 @@ services:
                 -Dmetacat.parentChildRelationshipProperties.createEnabled=true
                 -Dmetacat.parentChildRelationshipProperties.getEnabled=true
                 -Dmetacat.parentChildRelationshipProperties.renameEnabled=true
-                -Dmetacat.parentChildRelationshipProperties.dropEnabled=true'
+                -Dmetacat.parentChildRelationshipProperties.dropEnabled=true
+                -Dmetacat.partitionColumnProperties.omitVoidPartitionEnabled=true'
         labels:
           - "com.netflix.metacat.oss.test"
           - "com.netflix.metacat.oss.test.war"