From 1ad774d49ec7be0a1a8a1a71a6293260ccfad90a Mon Sep 17 00:00:00 2001 From: Lin Liu Date: Fri, 17 Jan 2025 13:19:36 -0800 Subject: [PATCH] Enable log file only test for Hive MOR tables --- .../table/action/commit/BaseJavaCommitActionExecutor.java | 5 +++++ .../hudi/hadoop/TestHoodieFileGroupReaderOnHive.java | 7 ------- .../common/table/read/TestHoodieFileGroupReaderBase.java | 7 ++++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java index 2a9c2b86024f5..f38de6a18ea84 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/BaseJavaCommitActionExecutor.java @@ -36,6 +36,7 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.execution.JavaLazyInsertIterable; +import org.apache.hudi.io.AppendHandleFactory; import org.apache.hudi.io.CreateHandleFactory; import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.io.HoodieMergeHandleFactory; @@ -268,6 +269,10 @@ public Iterator> handleInsert(String idPfx, Iterator) Collections.EMPTY_LIST).iterator(); } + if (table.getIndex().canIndexLogFiles()) { + return new JavaLazyInsertIterable<>(recordItr, true, config, instantTime, table, idPfx, + taskContextSupplier, new AppendHandleFactory()); + } return new JavaLazyInsertIterable<>(recordItr, true, config, instantTime, table, idPfx, taskContextSupplier, new CreateHandleFactory<>()); } diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/hadoop/TestHoodieFileGroupReaderOnHive.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/hadoop/TestHoodieFileGroupReaderOnHive.java index 93848f3496a2c..d5125349ed711 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/hadoop/TestHoodieFileGroupReaderOnHive.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/hadoop/TestHoodieFileGroupReaderOnHive.java @@ -67,7 +67,6 @@ import org.apache.hadoop.mapred.Reporter; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import java.io.IOException; import java.util.ArrayList; @@ -87,12 +86,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieFileGroupReaderOnHive extends TestHoodieFileGroupReaderBase { - - @Override - @Disabled("[HUDI-8072]") - public void testReadLogFilesOnlyInMergeOnReadTable(RecordMergeMode recordMergeMode, String logDataBlockFormat) throws Exception { - } - private static final String PARTITION_COLUMN = "datestr"; private static JobConf baseJobConf; private static HdfsTestService hdfsTestService; diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java index fb1eddd83fbf6..5084238972d43 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java @@ -155,13 +155,14 @@ public void testReadLogFilesOnlyInMergeOnReadTable(RecordMergeMode recordMergeMo // Use InMemoryIndex to generate log only mor table writeConfigs.put("hoodie.index.type", "INMEMORY"); + try (HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(0xDEEF)) { - // One commit; reading one file group containing a base file only - commitToTable(dataGen.generateInserts("001", 100), INSERT.value(), writeConfigs); + // One commit: reading one file group containing 1 log file only. + commitToTable(dataGen.generateInserts("001", 100), UPSERT.value(), writeConfigs); validateOutputFromFileGroupReader( getStorageConf(), getBasePath(), dataGen.getPartitionPaths(), false, 1, recordMergeMode); - // Two commits; reading one file group containing a base file and a log file + // Two commits: reading one file group with 2 log files only. commitToTable(dataGen.generateUpdates("002", 100), UPSERT.value(), writeConfigs); validateOutputFromFileGroupReader( getStorageConf(), getBasePath(), dataGen.getPartitionPaths(), false, 2, recordMergeMode);