[fix](group commit) Add retry when delete bitmap lock expired on group commit (#37600)

group commit writing mow table may encouter delete bitmap lock expired,
add retry mechanism to avoid group commit fail.

Author: hust-hhb

be/src/runtime/group_commit_mgr.cpp

@@ -23,6 +23,7 @@
 #include <chrono>
 
 #include "client_cache.h"
+#include "cloud/config.h"
 #include "common/compiler_util.h"
 #include "common/config.h"
 #include "common/status.h"
@@ -446,13 +447,26 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_
         }
         TLoadTxnCommitResult result;
         TNetworkAddress master_addr = _exec_env->master_info()->network_address;
-        st = ThriftRpcHelper::rpc<FrontendServiceClient>(
-                master_addr.hostname, master_addr.port,
-                [&request, &result](FrontendServiceConnection& client) {
-                    client->loadTxnCommit(result, request);
-                },
-                10000L);
-        result_status = Status::create(result.status);
+        int retry_times = 0;
+        while (retry_times < config::mow_stream_load_commit_retry_times) {
+            st = ThriftRpcHelper::rpc<FrontendServiceClient>(
+                    master_addr.hostname, master_addr.port,
+                    [&request, &result](FrontendServiceConnection& client) {
+                        client->loadTxnCommit(result, request);
+                    },
+                    10000L);
+            result_status = Status::create(result.status);
+            // DELETE_BITMAP_LOCK_ERROR will be retried
+            if (result_status.ok() || !result_status.is<ErrorCode::DELETE_BITMAP_LOCK_ERROR>()) {
+                break;
+            }
+            LOG_WARNING("Failed to commit txn on group commit")
+                    .tag("label", label)
+                    .tag("txn_id", txn_id)
+                    .tag("retry_times", retry_times)
+                    .error(result_status);
+            retry_times++;
+        }
     } else {
         // abort txn
         TLoadTxnRollbackRequest request;