[fix](dynamic partition) drop partition exclude history_partition_num (…

…apache#37539)

FIX:   

When dropping dynamic partition, PR apache#35778 will use math.max(start,
-history_partition_num) as the first partition, but it may delete users'
partitions if they specify both start and history_partition_num
inappropriately. For safety reason, revert this behavious changed, only
use start as the first partition when dropping partitions.

For those who had specified a very small start value, drop partitions
will catch an exception , and stop dropping this table's partition and
then record this error in dynamic info. Users can use command `SHOW
DYNAMIC PARTITION TABLES FROM DBXXX` to know this error. From this
error, it will give user hint to modify start if they really specify a
error start.

---------

Co-authored-by: Yongqiang YANG <[email protected]>

fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java

@@ -448,8 +448,10 @@ private ArrayList<DropPartitionClause> getDropPartitionClause(Database db, OlapT
             return dropPartitionClauses;
         }
 
-        int realStart = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(),
-                dynamicPartitionProperty.getHistoryPartitionNum());
+        // drop partition only considering start, not considering history_partition_num.
+        // int realStart = DynamicPartitionUtil.getRealStart(dynamicPartitionProperty.getStart(),
+        //         dynamicPartitionProperty.getHistoryPartitionNum());
+        int realStart = dynamicPartitionProperty.getStart();
         ZonedDateTime now = ZonedDateTime.now(dynamicPartitionProperty.getTimeZone().toZoneId());
         String lowerBorder = DynamicPartitionUtil.getPartitionRangeString(dynamicPartitionProperty,
                 now, realStart, partitionFormat);
@@ -464,9 +466,12 @@ private ArrayList<DropPartitionClause> getDropPartitionClause(Database db, OlapT
         } catch (Exception e) {
             // AnalysisException: keys.size is always equal to column.size, cannot reach this exception
             // IllegalArgumentException: lb is greater than ub
-            LOG.warn("Error in gen reservePartitionKeyRange. db: {}, table: {}",
-                    db.getFullName(), olapTable.getName(), e);
-            recordDropPartitionFailedMsg(db.getFullName(), olapTable.getName(), e.getMessage(), olapTable.getId());
+            String hint = "'dynamic_partition.start' = " + dynamicPartitionProperty.getStart()
+                    + ", maybe it's too small, can use alter table sql to increase it. ";
+            LOG.warn("Error in gen reservePartitionKeyRange. db: {}, table: {}. {}",
+                    db.getFullName(), olapTable.getName(), hint, e);
+            recordDropPartitionFailedMsg(db.getFullName(), olapTable.getName(), hint + e.getMessage(),
+                    olapTable.getId());
             return dropPartitionClauses;
         }
 
@@ -585,10 +590,12 @@ public void executeDynamicPartition(Collection<Pair<Long, Long>> dynamicPartitio
                     addPartitionClauses = getAddPartitionClause(db, olapTable, partitionColumn, partitionFormat,
                             executeFirstTime);
                 }
+                clearDropPartitionFailedMsg(olapTable.getId());
                 dropPartitionClauses = getDropPartitionClause(db, olapTable, partitionColumn, partitionFormat);
                 tableName = olapTable.getName();
             } catch (Exception e) {
-                LOG.warn("has error", e);
+                LOG.warn("db [{}-{}], table [{}-{}]'s dynamic partition has error",
+                        db.getId(), db.getName(), olapTable.getId(), olapTable.getName(), e);
                 if (executeFirstTime) {
                     throw new DdlException(e.getMessage());
                 }
@@ -602,10 +609,10 @@ public void executeDynamicPartition(Collection<Pair<Long, Long>> dynamicPartitio
                 }
                 try {
                     Env.getCurrentEnv().dropPartition(db, olapTable, dropPartitionClause);
-                    clearDropPartitionFailedMsg(olapTable.getId());
                 } catch (Exception e) {
                     recordDropPartitionFailedMsg(db.getFullName(), tableName, e.getMessage(), olapTable.getId());
-                    LOG.warn("has error", e);
+                    LOG.warn("db [{}-{}], table [{}-{}]'s dynamic partition has error",
+                            db.getId(), db.getName(), olapTable.getId(), olapTable.getName(), e);
                     if (executeFirstTime) {
                         throw new DdlException(e.getMessage());
                     }

fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java

@@ -744,7 +744,11 @@ public void testFillHistoryDynamicPartition3() throws Exception {
         String alter5 = "alter table test.dynamic_partition4 set ('dynamic_partition.history_partition_num' = '3')";
         ExceptionChecker.expectThrowsNoException(() -> alterTable(alter5));
         Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartitionFirstTime(db.getId(), tbl4.getId());
-        Assert.assertEquals(7, tbl4.getPartitionNames().size());
+        Assert.assertEquals(9, tbl4.getPartitionNames().size());
+        String dropPartitionErr = Env.getCurrentEnv().getDynamicPartitionScheduler()
+                .getRuntimeInfo(tbl4.getId(), DynamicPartitionScheduler.DROP_PARTITION_MSG);
+        Assert.assertTrue(dropPartitionErr.contains("'dynamic_partition.start' = -99999999, maybe it's too small, "
+                + "can use alter table sql to increase it."));
     }
 
     @Test

regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_failed.groovy

@@ -18,8 +18,8 @@
 suite('test_dynamic_partition_failed', 'nonConcurrent') {
     def old_max_dynamic_partition_num = getFeConfig('max_dynamic_partition_num')
     try {
-        sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_1'
-        sql '''CREATE TABLE test_dynamic_partition_failed_1
+        sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_ok1 FORCE'
+        sql '''CREATE TABLE test_dynamic_partition_failed_ok1
               ( `k1` datetime NULL )
               PARTITION BY RANGE (k1)()
               DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@@ -36,8 +36,13 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') {
                 "dynamic_partition.create_history_partition" = "true"
               )'''
 
-        def partitions = sql_return_maparray "SHOW PARTITIONS FROM test_dynamic_partition_failed_1"
+        def partitions = sql_return_maparray "SHOW PARTITIONS FROM test_dynamic_partition_failed_ok1"
         assertEquals(9, partitions.size());
+        def dynamicInfo = sql_return_maparray("SHOW DYNAMIC PARTITION TABLES").find { it.TableName == 'test_dynamic_partition_failed_ok1' }
+        logger.info("table dynamic info: " + dynamicInfo)
+        assertNotNull(dynamicInfo)
+        assertTrue(dynamicInfo.LastDropPartitionMsg.contains("'dynamic_partition.start' = -99999999, maybe it's too small, "
+                + "can use alter table sql to increase it."))
 
         setFeConfig('max_dynamic_partition_num', Integer.MAX_VALUE)
 
@@ -96,7 +101,7 @@ suite('test_dynamic_partition_failed', 'nonConcurrent') {
         }
     } finally {
         setFeConfig('max_dynamic_partition_num', old_max_dynamic_partition_num)
-        sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_1'
+        sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_ok1 FORCE'
         sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_2'
         sql 'DROP TABLE IF EXISTS test_dynamic_partition_failed_3'
     }