From ff0fecb705586e1577e87c1106398b6b39606b17 Mon Sep 17 00:00:00 2001 From: Yingjian Wu Date: Wed, 17 Apr 2024 22:24:26 -0400 Subject: [PATCH 1/3] test view --- .../netflix/metacat/MetacatSmokeSpec.groovy | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index 697d8d647..be9d60a5f 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -165,6 +165,119 @@ class MetacatSmokeSpec extends Specification { thrown(MetacatNotSupportedException) } + @Unroll + def "Test create/get view with nested fields with upper case"() { + given: + def catalogName = 'embedded-fast-hive-metastore' + def databaseName = 'iceberg_db' + def tableName = 'iceberg_table_with_upper_case_nested_fields' + def uri = isLocalEnv ? String.format('file:/tmp/data/') : null + def tableDto = new TableDto( + name: QualifiedName.ofTable(catalogName, databaseName, tableName), + serde: new StorageDto( + owner: 'metacat-test', + inputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + serializationLib: 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe', + parameters: [ + 'serialization.format': '1' + ], + uri: uri + ), + definitionMetadata: null, + dataMetadata: null, + fields: [ + new FieldDto( + comment: null, + name: "dateint", + pos: 0, + type: "long", + partition_key: true, + ), + new FieldDto( + comment: null, + name: "info", + pos: 1, + partition_key: false, + type: "(name: chararray, address: (NAME: chararray), nestedArray: {(FIELD1: chararray, field2: chararray)})", + ) + ] + ) + + def metadataLocation = String.format('/tmp/data/metadata/00000-0b60cc39-438f-413e-96c2-2694d7926529.metadata.json') + + if (isIcebergTable) { + def metadata = [common_view: 'true', metadata_location: metadataLocation] + tableDto.setMetadata(metadata) + } + when: + try {api.createDatabase(catalogName, databaseName, new DatabaseCreateRequestDto()) + } catch (Exception ignored) { + } + api.createTable(catalogName, databaseName, tableName, tableDto) + def tableDTO = api.getTable(catalogName, databaseName, tableName, true, true, true) + + then: + noExceptionThrown() + if (isIcebergTable) { + tableDTO.metadata.get("metadata_location").equals(metadataLocation) + } + tableDTO.getFields().size() == 2 + def nestedFieldDto = tableDTO.getFields().find { it.name == "info" } + // assert that the type field also keeps the name fidelity + assert nestedFieldDto.type == "(name: chararray,address: (NAME: chararray),nestedArray: {(FIELD1: chararray,field2: chararray)})" : "The type differ from the expected. They are: $nestedFieldDto.type" + + // assert that the json representation keeps the name fidelity + def expectedJsonString = """ + { + "type": "row", + "fields": [ + { + "name": "name", + "type": "chararray" + }, + { + "name": "address", + "type": { + "type": "row", + "fields": [ + { + "name": "NAME", + "type": "chararray" + } + ] + } + }, + { + "name": "nestedArray", + "type": { + "type": "array", + "elementType": { + "type": "row", + "fields": [ + { + "name": "FIELD1", + "type": "chararray" + }, + { + "name": "field2", + "type": "chararray" + } + ] + } + } + } + ] + } + """ + + JSONAssert.assertEquals(nestedFieldDto.jsonType.toString(), expectedJsonString, false) + cleanup: + api.deleteTable(catalogName, databaseName, tableName) + where: + isIcebergTable << [true] + } + @Unroll def "Test create/get table with nested fields with upper case"() { given: From 3bb48bdb021e2176bc3566ecc2d2627cc1a4e862 Mon Sep 17 00:00:00 2001 From: Yingjian Wu Date: Wed, 17 Apr 2024 22:43:03 -0400 Subject: [PATCH 2/3] test view --- .../connector/hive/converters/HiveConnectorInfoConverter.java | 1 + .../groovy/com/netflix/metacat/MetacatSmokeSpec.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java b/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java index 1b2f02b0f..4693cb578 100644 --- a/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java +++ b/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java @@ -134,6 +134,7 @@ public TableInfo toTableInfo(final QualifiedName name, final Table table) { // add the data fields to the nonPartitionColumns //ignore all exceptions try { + assert false : "this should fail"; if (nonPartitionColumns.isEmpty()) { for (StructField field : HiveTableUtil.getTableStructFields(table)) { final FieldSchema fieldSchema = new FieldSchema(field.getFieldName(), diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index be9d60a5f..07541e323 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -272,6 +272,7 @@ class MetacatSmokeSpec extends Specification { """ JSONAssert.assertEquals(nestedFieldDto.jsonType.toString(), expectedJsonString, false) + JSONAssert.assertEquals("", expectedJsonString, false) cleanup: api.deleteTable(catalogName, databaseName, tableName) where: From e55c3699c6089facbd7d98129e3086e22a287841 Mon Sep 17 00:00:00 2001 From: Yingjian Wu Date: Wed, 17 Apr 2024 22:52:04 -0400 Subject: [PATCH 3/3] test view --- .../connector/hive/converters/HiveConnectorInfoConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java b/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java index 4693cb578..3237e247e 100644 --- a/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java +++ b/metacat-connector-hive/src/main/java/com/netflix/metacat/connector/hive/converters/HiveConnectorInfoConverter.java @@ -131,10 +131,10 @@ public Database fromDatabaseInfo(final DatabaseInfo databaseInfo) { public TableInfo toTableInfo(final QualifiedName name, final Table table) { final List nonPartitionColumns = (table.getSd() != null) ? table.getSd().getCols() : Collections.emptyList(); + log.debug("Hey Size = " + nonPartitionColumns.size()); // add the data fields to the nonPartitionColumns //ignore all exceptions try { - assert false : "this should fail"; if (nonPartitionColumns.isEmpty()) { for (StructField field : HiveTableUtil.getTableStructFields(table)) { final FieldSchema fieldSchema = new FieldSchema(field.getFieldName(),