Skip auto analyze empty table. (#43865)

apache · Nov 15, 2024 · c2320e2 · c2320e2
1 parent 466c309
commit c2320e2
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -219,6 +219,9 @@ public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlExce
             }
             List<AnalysisInfo> jobs = new ArrayList<>();
             autoCollector.createAnalyzeJobForTbl(stmt.getDb(), jobs, stmt.getTable());
+            if (jobs.isEmpty()) {
+                return;
+            }
             AnalysisInfo job = autoCollector.getReAnalyzeRequiredPart(jobs.get(0));
             if (job != null) {
                 Env.getCurrentEnv().getStatisticsAutoCollector().createSystemAnalysisJob(job);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -144,7 +144,6 @@ protected List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<? extends TableIf>
             } catch (Throwable t) {
                 LOG.warn("Failed to analyze table {}.{}.{}",
                         db.getCatalog().getName(), db.getFullName(), table.getName(), t);
-                continue;
             }
         }
         return analysisInfos;
@@ -186,7 +185,19 @@ protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
                 return;
             }
         }
-        long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 : table.getRowCount();
+        // We don't auto analyze empty table to avoid all 0 stats.
+        // Because all 0 is more dangerous than unknown stats when row count report is delayed.
+        AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+        TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId());
+        long rowCount = table.getRowCount();
+        if (rowCount <= 0) {
+            LOG.info("Table {} is empty, remove its old stats and skip auto analyze it.", table.getName());
+            // Remove the table's old stats if exists.
+            if (tableStatsStatus != null && !tableStatsStatus.isColumnsStatsEmpty()) {
+                manager.dropStats(table);
+            }
+            return;
+        }
         AnalysisInfo jobInfo = new AnalysisInfoBuilder()
                 .setJobId(Env.getCurrentEnv().getNextId())
                 .setCatalogId(db.getCatalog().getId())

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -242,4 +242,8 @@ protected void clearStaleIndexRowCount(OlapTable table) {
     protected void addIndexRowForTest(long indexId, long rowCount) {
         indexesRowCount.put(indexId, rowCount);
     }
+
+    public boolean isColumnsStatsEmpty() {
+        return colNameToColStatsMeta == null || colNameToColStatsMeta.isEmpty();
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -142,6 +142,11 @@ public List<Column> getSchemaAllIndexes(boolean full) {
                 columns.add(new Column("c2", PrimitiveType.HLL));
                 return columns;
             }
+
+            @Mock
+            public long getRowCount() {
+                return 1;
+            }
         };
         StatisticsAutoCollector saa = new StatisticsAutoCollector();
         List<AnalysisInfo> analysisInfoList = saa.constructAnalysisInfo(new Database(1, "anydb"));
@@ -397,6 +402,11 @@ public List<Long> getMvColumnIndexIds(String columnName) {
                 objects.add(-1L);
                 return objects;
             }
+
+            @Mock
+            public long getRowCount() {
+                return 1;
+            }
         };
 
         new MockUp<StatisticsUtil>() {
@@ -469,6 +479,11 @@ public List<Long> getMvColumnIndexIds(String columnName) {
                 objects.add(-1L);
                 return objects;
             }
+
+            @Mock
+            public long getRowCount() {
+                return 1;
+            }
         };
 
         new MockUp<StatisticsUtil>() {

diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -689,6 +689,18 @@ suite("test_analyze_mv") {
         assertEquals("0", result_row[0][3])
         assertEquals("-1", result_row[0][4])
 
+        // ** Embedded test for skip auto analyze when table is empty
+        sql """analyze table mvTestDup properties ("use.auto.analyzer" = "true")"""
+        def empty_test = sql """show auto analyze mvTestDup"""
+        assertEquals(0, empty_test.size())
+        empty_test = sql """show column stats mvTestDup"""
+        assertEquals(0, empty_test.size())
+        // ** End of embedded test
+
+        sql """analyze table mvTestDup with sync"""
+        empty_test = sql """show column stats mvTestDup"""
+        assertEquals(12, empty_test.size())
+
         for (int i = 0; i < 120; i++) {
             result_row = sql """show index stats mvTestDup mv3"""
             logger.info("mv3 stats: " + result_row)
@@ -703,6 +715,27 @@ suite("test_analyze_mv") {
         assertEquals("mv3", result_row[0][1])
         assertEquals("0", result_row[0][3])
         assertEquals("0", result_row[0][4])
+
+        // ** Embedded test for skip auto analyze when table is empty again
+        sql """analyze table mvTestDup properties ("use.auto.analyzer" = "true")"""
+        empty_test = sql """show auto analyze mvTestDup"""
+        assertEquals(0, empty_test.size())
+        empty_test = sql """show column stats mvTestDup"""
+        for (int i = 0; i < 100; i++) {
+            empty_test = sql """show column stats mvTestDup"""
+            if (empty_test.size() != 0) {
+                logger.info("async delete is not finished yet.")
+                Thread.sleep(1000)
+            }
+            break
+        }
+        assertEquals(0, empty_test.size())
+        // ** End of embedded test
+
+        sql """analyze table mvTestDup with sync"""
+        empty_test = sql """show column stats mvTestDup"""
+        assertEquals(12, empty_test.size())
+
         sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);"""
         result_row = sql """show index stats mvTestDup mv3"""
         assertEquals(1, result_row.size())