update

Author: yujun777

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -2533,12 +2533,6 @@ public class Config extends ConfigBase {
     })
     public static long analyze_record_limit = 20000;
 
-    @ConfField(mutable = true, masterOnly = true, description = {
-            "Auto Buckets中预估的压缩数据的倍率",
-            "the estimated compress factor of partition size in Auto Buckets"
-    })
-    public static int autobucket_compress_size_factor = 5;
-
     @ConfField(mutable = true, masterOnly = true, description = {
             "Auto Buckets中最小的buckets数目",
             "min buckets of auto bucket"

fe/fe-core/src/main/java/org/apache/doris/clone/BeLoadRebalancer.java

@@ -113,14 +113,17 @@ protected List<TabletSchedCtx> selectAlternativeTabletsForCluster(
                 numOfLowPaths += pathSlot.getTotalAvailBalanceSlotNum();
             }
         }
-        LOG.info("get number of low load paths: {}, with medium: {}", numOfLowPaths, medium);
+        LOG.info("get number of low load paths: {}, with medium: {}, tag: {}, isUrgent {}",
+                numOfLowPaths, medium, clusterStat.getTag(), isUrgent);
 
         List<String> alternativeTabletInfos = Lists.newArrayList();
         int clusterAvailableBEnum = infoService.getAllBackendIds(true).size();
         List<Set<Long>> lowBETablets = lowBEs.stream()
                 .map(beStat -> Sets.newHashSet(invertedIndex.getTabletIdsByBackendId(beStat.getBeId())))
                 .collect(Collectors.toList());
 
+        boolean hasCandidateTablet = false;
+
         // choose tablets from high load backends.
         // BackendLoadStatistic is sorted by load score in ascend order,
         // so we need to traverse it from last to first
@@ -222,6 +225,8 @@ protected List<TabletSchedCtx> selectAlternativeTabletsForCluster(
                         continue;
                     }
 
+                    hasCandidateTablet = true;
+
                     // for urgent disk, pick tablets order by size,
                     // then it may always pick tablets that was on the low backends.
                     if (!lowBETablets.isEmpty()
@@ -270,6 +275,9 @@ protected List<TabletSchedCtx> selectAlternativeTabletsForCluster(
         if (!alternativeTablets.isEmpty()) {
             LOG.info("select alternative tablets, medium: {}, is urgent: {}, num: {}, detail: {}",
                     medium, isUrgent, alternativeTablets.size(), alternativeTabletInfos);
+        } else if (isUrgent && !hasCandidateTablet) {
+            LOG.info("urgent balance cann't found candidate tablets. medium: {}, tag: {}",
+                    medium, clusterStat.getTag());
         }
         return alternativeTablets;
     }

fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java

@@ -234,9 +234,7 @@ private static int getBucketsNum(DynamicPartitionProperty property, OlapTable ta
                 .collect(Collectors.toCollection(ArrayList::new));
         long estimatePartitionSize = getNextPartitionSize(partitionSizeArray);
         // plus 5 for uncompressed data
-        // replica's actual disk usage is a litter bigger then its reported data size
-        // but 5 times maybe a little too big, i don't known why use so big. just add a config here.
-        long uncompressedPartitionSize = estimatePartitionSize * Config.autobucket_compress_size_factor;
+        long uncompressedPartitionSize = estimatePartitionSize * 5;
         int bucketsNum = AutoBucketUtils.getBucketsNum(uncompressedPartitionSize, Config.autobucket_min_buckets);
         LOG.info("autobucket calc with {} history partitions, table: [{}-{}], partition: {}, buckets num: {}, "
                 + " estimate partition size: {}, last partitions(partition name, local size, remote size): {}",

fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java

@@ -1766,20 +1766,25 @@ public void testAutoBuckets() throws Exception {
 
         table.readLock();
         try {
-            // first 40 partitions with size 0,  then 10 partitions with size 20GB
-            for (int i = 0; i < 50; i++) {
+            // first 40 partitions with size 0,  then 13 partitions with size 100GB(10GB * 10 buckets)
+            for (int i = 0; i < 52; i++) {
                 Partition partition = partitions.get(i);
                 partition.updateVisibleVersion(2L);
                 for (MaterializedIndex idx : partition.getMaterializedIndices(
                         MaterializedIndex.IndexExtState.VISIBLE)) {
+                    Assert.assertEquals(10, idx.getTablets().size());
                     for (Tablet tablet : idx.getTablets()) {
                         for (Replica replica : tablet.getReplicas()) {
                             replica.updateVersion(2L);
-                            replica.setDataSize(i < 40 ? 0L : 20L << 30);
+                            replica.setDataSize(i < 40 ? 0L : 10L << 30);
                             replica.setRowCount(1000L);
                         }
                     }
                 }
+                if (i >= 40) {
+                    // first 52 partitions are 10 buckets(FeConstants.default_bucket_num)
+                    Assert.assertEquals(10 * (10L << 30), partition.getAllDataSize(true));
+                }
             }
         } finally {
             table.readUnlock();
@@ -1793,6 +1798,7 @@ public void testAutoBuckets() throws Exception {
         partitions = Lists.newArrayList(table.getAllPartitions());
         partitions.sort(Comparator.comparing(Partition::getId));
         Assert.assertEquals(54, partitions.size());
-        Assert.assertTrue(partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum() > 40);
+        // 100GB total, 1GB per bucket, should 100 buckets.
+        Assert.assertEquals(100, partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum());
     }
 }