log restore: fix the bottleneck of btree map (#59896)

close #59900

Author: 3pointer

br/pkg/restore/log_client/client_test.go

@@ -1513,15 +1513,20 @@ func TestLogSplitStrategy(t *testing.T) {
 	// Create a split client with the mock PD client.
 	client := split.NewClient(mockPDCli, nil, nil, 100, 4)
 
+	// these files should skip accumulation
+	smallFiles := make([]*backuppb.DataFileInfo, 0, 10)
+	for j := 0; j < 20; j++ {
+		smallFiles = append(smallFiles, fakeFile(1, 100, 1024*1024, 100))
+	}
+
 	// Define a mock iterator with sample data files.
-	mockIter := iter.FromSlice([]*backuppb.DataFileInfo{
-		fakeFile(1, 100, 100, 100),
+	mockIter := iter.FromSlice(append(smallFiles, []*backuppb.DataFileInfo{
 		fakeFile(1, 200, 2*units.MiB, 200),
 		fakeFile(2, 100, 3*units.MiB, 300),
 		fakeFile(3, 100, 10*units.MiB, 100000),
 		fakeFile(1, 300, 3*units.MiB, 10),
 		fakeFile(1, 400, 4*units.MiB, 10),
-	})
+	}...))
 	logIter := toLogDataFileInfoIter(mockIter)
 
 	// Initialize a wrapper for the file restorer with a region splitter.
@@ -1548,7 +1553,7 @@ func TestLogSplitStrategy(t *testing.T) {
 	count := 0
 	for i := helper.TryNext(ctx); !i.Finished; i = helper.TryNext(ctx) {
 		require.NoError(t, i.Err)
-		if count == expectSplitCount {
+		if count == len(smallFiles)+expectSplitCount {
 			// Verify that no split occurs initially due to insufficient data.
 			regions, err := mockPDCli.ScanRegions(ctx, []byte{}, []byte{}, 0)
 			require.NoError(t, err)
@@ -1562,6 +1567,9 @@ func TestLogSplitStrategy(t *testing.T) {
 		count += 1
 	}
 
+	// iterate 20 small files + 4 valid files
+	require.Equal(t, len(smallFiles)+4, count)
+
 	// Verify that a split occurs on the second region due to excess data.
 	regions, err := mockPDCli.ScanRegions(ctx, []byte{}, []byte{}, 0)
 	require.NoError(t, err)

br/pkg/restore/log_client/log_split_strategy.go

@@ -63,25 +63,27 @@ func NewLogSplitStrategy(
 const splitFileThreshold = 1024 * 1024 // 1 MB
 
 func (ls *LogSplitStrategy) Accumulate(file *LogDataFileInfo) {
+	// skip accumulate file less than 1MB. to prevent too much split & scatter occurs
+	// and protect the performance of BTreeMap
 	if file.Length > splitFileThreshold {
 		ls.AccumulateCount += 1
-	}
-	splitHelper, exist := ls.TableSplitter[file.TableId]
-	if !exist {
-		splitHelper = split.NewSplitHelper()
-		ls.TableSplitter[file.TableId] = splitHelper
-	}
+		splitHelper, exist := ls.TableSplitter[file.TableId]
+		if !exist {
+			splitHelper = split.NewSplitHelper()
+			ls.TableSplitter[file.TableId] = splitHelper
+		}
 
-	splitHelper.Merge(split.Valued{
-		Key: split.Span{
-			StartKey: file.StartKey,
-			EndKey:   file.EndKey,
-		},
-		Value: split.Value{
-			Size:   file.Length,
-			Number: file.NumberOfEntries,
-		},
-	})
+		splitHelper.Merge(split.Valued{
+			Key: split.Span{
+				StartKey: file.StartKey,
+				EndKey:   file.EndKey,
+			},
+			Value: split.Value{
+				Size:   file.Length,
+				Number: file.NumberOfEntries,
+			},
+		})
+	}
 }
 
 func (ls *LogSplitStrategy) ShouldSplit() bool {