lightning: redact external storage url (pingcap#59256) (pingcap#59380)

close pingcap#59086

Author: ti-chi-bot

br/pkg/lightning/config/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//br/pkg/lightning/log",
         "//br/pkg/version/build",
         "//pkg/config",
+        "//pkg/parser/ast",
         "//pkg/parser/mysql",
         "//pkg/util",
         "//pkg/util/mathutil",
@@ -43,7 +44,7 @@ go_test(
     ],
     embed = [":config"],
     flaky = True,
-    shard_count = 48,
+    shard_count = 49,
     deps = [
         "//br/pkg/lightning/common",
         "@com_github_burntsushi_toml//:toml",

br/pkg/lightning/config/config.go

@@ -37,6 +37,7 @@ import (
 	"github.com/pingcap/tidb/br/pkg/lightning/common"
 	"github.com/pingcap/tidb/br/pkg/lightning/log"
 	tidbcfg "github.com/pingcap/tidb/pkg/config"
+	"github.com/pingcap/tidb/pkg/parser/ast"
 	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/util"
 	"github.com/pingcap/tidb/pkg/util/mathutil"
@@ -272,6 +273,16 @@ func (cfg *Config) String() string {
 	return string(bytes)
 }
 
+// Redact redacts the sensitive information.
+func (cfg *Config) Redact() string {
+	originDir := cfg.Mydumper.SourceDir
+	defer func() {
+		cfg.Mydumper.SourceDir = originDir
+	}()
+	cfg.Mydumper.SourceDir = ast.RedactURL(cfg.Mydumper.SourceDir)
+	return cfg.String()
+}
+
 // ToTLS creates a common.TLS from the config.
 func (cfg *Config) ToTLS() (*common.TLS, error) {
 	hostPort := net.JoinHostPort(cfg.TiDB.Host, strconv.Itoa(cfg.TiDB.StatusPort))

br/pkg/lightning/config/config_test.go

@@ -1322,3 +1322,30 @@ func TestAdjustConflict(t *testing.T) {
 	cfg.Conflict.MaxRecordRows = 1
 	require.ErrorContains(t, cfg.Conflict.adjust(&cfg.TikvImporter, &cfg.App), `cannot record duplication (conflict.max-record-rows > 0) when use tikv-importer.backend = "tidb" and conflict.strategy = "replace"`)
 }
+
+func TestRedactConfig(t *testing.T) {
+	tests := []struct {
+		origin string
+		redact string
+	}{
+		{"", ""},
+		{":", ":"},
+		{"~/file", "~/file"},
+		{"gs://bucket/file", "gs://bucket/file"},
+		{"gs://bucket/file?access-key=123", "gs://bucket/file?access-key=123"},
+		{"gs://bucket/file?secret-access-key=123", "gs://bucket/file?secret-access-key=123"},
+		{"s3://bucket/file", "s3://bucket/file"},
+		{"s3://bucket/file?other-key=123", "s3://bucket/file?other-key=123"},
+		{"s3://bucket/file?access-key=123", "s3://bucket/file?access-key=xxxxxx"},
+		{"s3://bucket/file?secret-access-key=123", "s3://bucket/file?secret-access-key=xxxxxx"},
+		{"s3://bucket/file?access_key=123", "s3://bucket/file?access_key=xxxxxx"},
+		{"s3://bucket/file?secret_access_key=123", "s3://bucket/file?secret_access_key=xxxxxx"},
+	}
+	for _, tt := range tests {
+		cfg := NewConfig()
+		cfg.Mydumper.SourceDir = tt.origin
+
+		require.Contains(t, cfg.Redact(), tt.redact)
+		require.Contains(t, cfg.String(), tt.origin)
+	}
+}

br/pkg/lightning/lightning.go

@@ -429,7 +429,7 @@ func getKeyspaceName(db *sql.DB) (string, error) {
 
 func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, o *options) (err error) {
 	build.LogInfo(build.Lightning)
-	o.logger.Info("cfg", zap.Stringer("cfg", taskCfg))
+	o.logger.Info("cfg", zap.String("cfg", taskCfg.Redact()))
 
 	utils.LogEnvVariables()