objstore: retry on "server sent GOAWAY" http2 error (#60144) (#61938)

close #60143

Author: ti-chi-bot

br/pkg/storage/gcs.go

@@ -514,6 +514,9 @@ func shouldRetry(err error) bool {
 	retryableErrMsg := []string{
 		"http2: client connection force closed via ClientConn.Close",
 		"broken pipe",
+		"http2: client connection lost",
+		// See https://stackoverflow.com/questions/45209168/http2-server-sent-goaway-and-closed-the-connection-laststreamid-1999 for details.
+		"http2: server sent GOAWAY",
 	}
 
 	for _, msg := range retryableErrMsg {

br/pkg/storage/gcs_test.go

@@ -6,11 +6,13 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	goerrors "errors"
 	"flag"
 	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"testing"
 	"time"
@@ -599,3 +601,8 @@ func TestCtxUsage(t *testing.T) {
 	// before the fix, it's context canceled error
 	require.ErrorContains(t, err, "invalid_request")
 }
+
+func TestGCSShouldRetry(t *testing.T) {
+	require.True(t, shouldRetry(&url.Error{Err: goerrors.New("http2: server sent GOAWAY and closed the connectiont"), Op: "Get", URL: "https://storage.googleapis.com/storage/v1/"}))
+	require.True(t, shouldRetry(&url.Error{Err: goerrors.New("http2: client connection lost"), Op: "Get", URL: "https://storage.googleapis.com/storage/v1/"}))
+}