logutil: adapt new redact functions (#51642)

xhebox · web-flow · commit 15947c1cf2f7 · 2024-03-12T09:06:39.000Z
close #51310, close #51641
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -5806,13 +5806,13 @@ def go_deps():
         name = "com_github_pingcap_errors",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/errors",
-        sha256 = "b4db3d3c222d9039c84baacbbd9c46aa0346f3f04d2577a77475a64ecfefebf9",
-        strip_prefix = "github.com/pingcap/errors@v0.11.5-0.20231212100244-799fae176cfb",
+        sha256 = "0edb07dbd73a90f97e06e11e54b270d64d5cabe6142025682d840fe302087b23",
+        strip_prefix = "github.com/pingcap/errors@v0.11.5-0.20240311024730-e056997136bb",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20231212100244-799fae176cfb.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20240311024730-e056997136bb.zip",
         ],
     )
     go_repository(
diff --git a/br/pkg/redact/redact.go b/br/pkg/redact/redact.go
@@ -11,12 +11,17 @@ import (
 
 // InitRedact inits the enableRedactLog
 func InitRedact(redactLog bool) {
-	errors.RedactLogEnabled.Store(redactLog)
+	mode := errors.RedactLogDisable
+	if redactLog {
+		mode = errors.RedactLogEnable
+	}
+	errors.RedactLogEnabled.Store(mode)
 }
 
 // NeedRedact returns whether to redact log
 func NeedRedact() bool {
-	return errors.RedactLogEnabled.Load()
+	mode := errors.RedactLogEnabled.Load()
+	return mode != errors.RedactLogDisable && mode != ""
 }
 
 // String receives string argument and return omitted information if redact log enabled
diff --git a/go.mod b/go.mod
@@ -81,7 +81,7 @@ require (
 	github.com/otiai10/copy v1.2.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d
-	github.com/pingcap/errors v0.11.5-0.20231212100244-799fae176cfb
+	github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
 	github.com/pingcap/fn v1.0.0
 	github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f
diff --git a/go.sum b/go.sum
@@ -706,8 +706,8 @@ github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d/go.mod h1:p8QnkZn
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
-github.com/pingcap/errors v0.11.5-0.20231212100244-799fae176cfb h1:yqyP+k0mgRPpXJQDOCrtaG2YZym0ZDD+vt5JzlBUkrw=
-github.com/pingcap/errors v0.11.5-0.20231212100244-799fae176cfb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
+github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk=
+github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
 github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=
 github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
 github.com/pingcap/fn v1.0.0 h1:CyA6AxcOZkQh52wIqYlAmaVmF6EvrcqFywP463pjA8g=
diff --git a/pkg/executor/set_test.go b/pkg/executor/set_test.go
@@ -464,10 +464,10 @@ func TestSetVar(t *testing.T) {
 	tk.MustQuery(`select @@session.tidb_redact_log;`).Check(testkit.Rows("ON"))
 	tk.MustExec("set session tidb_redact_log = oFf")
 	tk.MustQuery(`select @@session.tidb_redact_log;`).Check(testkit.Rows("OFF"))
-	tk.MustExec("set session tidb_redact_log = On")
-	tk.MustQuery(`select @@session.tidb_redact_log;`).Check(testkit.Rows("ON"))
 	tk.MustExec("set session tidb_redact_log = marker")
 	tk.MustQuery(`select @@session.tidb_redact_log;`).Check(testkit.Rows("MARKER"))
+	tk.MustExec("set session tidb_redact_log = On")
+	tk.MustQuery(`select @@session.tidb_redact_log;`).Check(testkit.Rows("ON"))
 
 	tk.MustQuery("select @@tidb_dml_batch_size;").Check(testkit.Rows("0"))
 	tk.MustExec("set @@session.tidb_dml_batch_size = 120")
diff --git a/pkg/executor/test/admintest/BUILD.bazel b/pkg/executor/test/admintest/BUILD.bazel
@@ -30,6 +30,7 @@ go_test(
         "//pkg/util/logutil",
         "//pkg/util/logutil/consistency",
         "//pkg/util/mock",
+        "//pkg/util/redact",
         "@com_github_pingcap_errors//:errors",
         "@com_github_stretchr_testify//require",
         "@com_github_tikv_client_go_v2//tikv",
diff --git a/pkg/executor/test/admintest/admin_test.go b/pkg/executor/test/admintest/admin_test.go
@@ -41,6 +41,7 @@ import (
 	"github.com/pingcap/tidb/pkg/util/logutil"
 	"github.com/pingcap/tidb/pkg/util/logutil/consistency"
 	"github.com/pingcap/tidb/pkg/util/mock"
+	"github.com/pingcap/tidb/pkg/util/redact"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
@@ -1055,6 +1056,8 @@ func TestCheckFailReport(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := newInconsistencyKit(t, testkit.NewAsyncTestKit(t, store), newDefaultOpt())
 
+	rmode := tk.sctx.GetSessionVars().EnableRedactNew
+
 	// row more than unique index
 	func() {
 		defer tk.rebuild()
@@ -1072,7 +1075,7 @@ func TestCheckFailReport(t *testing.T) {
 		hook.Logs[0].CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "uk1"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		hook.Logs[0].CheckFieldNotEmpty(t, "row_mvcc")
 	}()
@@ -1094,7 +1097,7 @@ func TestCheckFailReport(t *testing.T) {
 		hook.Logs[0].CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "k2"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		hook.Logs[0].CheckFieldNotEmpty(t, "row_mvcc")
 	}()
@@ -1118,7 +1121,7 @@ func TestCheckFailReport(t *testing.T) {
 		logEntry.CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "k2"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		logEntry.CheckFieldNotEmpty(t, "row_mvcc")
 		logEntry.CheckFieldNotEmpty(t, "index_mvcc")
@@ -1162,7 +1165,7 @@ func TestCheckFailReport(t *testing.T) {
 		logEntry.CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "uk1"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		logEntry.CheckFieldNotEmpty(t, "row_mvcc")
 		logEntry.CheckFieldNotEmpty(t, "index_mvcc")
@@ -1205,7 +1208,7 @@ func TestCheckFailReport(t *testing.T) {
 		logEntry.CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "uk1"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		logEntry.CheckFieldNotEmpty(t, "row_mvcc")
 		logEntry.CheckFieldNotEmpty(t, "index_mvcc")
@@ -1231,7 +1234,7 @@ func TestCheckFailReport(t *testing.T) {
 		logEntry.CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "k2"),
-			zap.Stringer("row_id", kv.IntHandle(1)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(1))),
 		)
 		logEntry.CheckFieldNotEmpty(t, "row_mvcc")
 		logEntry.CheckFieldNotEmpty(t, "index_mvcc")
@@ -1271,7 +1274,7 @@ func TestCheckFailReport(t *testing.T) {
 		logEntry.CheckField(t,
 			zap.String("table_name", "admin_test"),
 			zap.String("index_name", "uk1"),
-			zap.Stringer("row_id", kv.IntHandle(282574488403969)),
+			zap.Stringer("row_id", redact.Stringer(rmode, kv.IntHandle(282574488403969))),
 		)
 		logEntry.CheckFieldNotEmpty(t, "row_mvcc")
 		logEntry.CheckFieldNotEmpty(t, "index_mvcc")
diff --git a/pkg/server/conn.go b/pkg/server/conn.go
@@ -1187,7 +1187,7 @@ func errStrForLog(err error, redactMode string) string {
 	} else {
 		ret = errors.ErrorStack(err)
 	}
-	return redact.Redact(redactMode, ret)
+	return redact.String(redactMode, ret)
 }
 
 func (cc *clientConn) addMetrics(cmd byte, startTime time.Time, err error) {
diff --git a/pkg/server/conn_stmt.go b/pkg/server/conn_stmt.go
@@ -583,7 +583,7 @@ func (cc *clientConn) preparedStmt2String(stmtID uint32) string {
 	}
 	sql := parser.Normalize(cc.preparedStmt2StringNoArgs(stmtID), sv.EnableRedactNew)
 	if m := sv.EnableRedactNew; m != "ON" {
-		sql += redact.Redact(sv.EnableRedactNew, sv.PlanCacheParams.String())
+		sql += redact.String(sv.EnableRedactNew, sv.PlanCacheParams.String())
 	}
 	return sql
 }
diff --git a/pkg/session/nontransactional.go b/pkg/session/nontransactional.go
@@ -69,7 +69,7 @@ type statementBuildInfo struct {
 }
 
 func (j job) String(redacted string) string {
-	return fmt.Sprintf("job id: %d, estimated size: %d, sql: %s", j.jobID, j.jobSize, redact.Redact(redacted, j.sql))
+	return fmt.Sprintf("job id: %d, estimated size: %d, sql: %s", j.jobID, j.jobSize, redact.String(redacted, j.sql))
 }
 
 // HandleNonTransactionalDML is the entry point for a non-transactional DML statement
diff --git a/pkg/session/session.go b/pkg/session/session.go
@@ -933,7 +933,7 @@ func (s *session) tryReplaceWriteConflictError(oldErr error) (newErr error) {
 	if !kv.ErrWriteConflict.Equal(oldErr) {
 		return nil
 	}
-	if errors.RedactLogEnabled.Load() {
+	if errors.RedactLogEnabled.Load() == errors.RedactLogEnable {
 		return nil
 	}
 	originErr := errors.Cause(oldErr)
@@ -1174,7 +1174,7 @@ func (s *session) retry(ctx context.Context, maxCnt uint) (err error) {
 				// We print the queries at the first try only.
 				sql := sqlForLog(st.GetTextToLog(false))
 				if sessVars.EnableRedactNew != "ON" {
-					sql += redact.Redact(sessVars.EnableRedactNew, sessVars.PlanCacheParams.String())
+					sql += redact.String(sessVars.EnableRedactNew, sessVars.PlanCacheParams.String())
 				}
 				logutil.Logger(ctx).Warn("retrying",
 					zap.Int64("schemaVersion", schemaVersion),
@@ -1669,7 +1669,7 @@ func (s *session) Parse(ctx context.Context, sql string) ([]ast.StmtNode, error)
 		// Only print log message when this SQL is from the user.
 		// Mute the warning for internal SQLs.
 		if !s.sessionVars.InRestrictedSQL {
-			logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.Redact(s.sessionVars.EnableRedactNew, sql)))
+			logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactNew, sql)))
 			s.sessionVars.StmtCtx.AppendError(err)
 		}
 		return nil, err
@@ -1719,7 +1719,7 @@ func (s *session) ParseWithParams(ctx context.Context, sql string, args ...any)
 	if err != nil {
 		s.rollbackOnError(ctx)
 		logSQL := sql[:min(500, len(sql))]
-		logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.Redact(s.sessionVars.EnableRedactNew, logSQL)))
+		logutil.Logger(ctx).Warn("parse SQL failed", zap.Error(err), zap.String("SQL", redact.String(s.sessionVars.EnableRedactNew, logSQL)))
 		return nil, util.SyntaxError(err)
 	}
 	durParse := time.Since(parseStartTime)
@@ -3935,7 +3935,7 @@ func logGeneralQuery(execStmt *executor.ExecStmt, s *session, isPrepared bool) {
 
 		query = executor.QueryReplacer.Replace(query)
 		if vars.EnableRedactNew != "ON" {
-			query += redact.Redact(vars.EnableRedactNew, vars.PlanCacheParams.String())
+			query += redact.String(vars.EnableRedactNew, vars.PlanCacheParams.String())
 		}
 		logutil.BgLogger().Info("GENERAL_LOG",
 			zap.Uint64("conn", vars.ConnectionID),
diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go
@@ -2154,7 +2154,7 @@ var defaultSysVars = []*SysVar{
 	{Scope: ScopeGlobal | ScopeSession, Name: TiDBRedactLog, Value: DefTiDBRedactLog, Type: TypeEnum, PossibleValues: []string{Off, On, Marker}, SetSession: func(s *SessionVars, val string) error {
 		s.EnableRedactLog = val != Off
 		s.EnableRedactNew = val
-		errors.RedactLogEnabled.Store(s.EnableRedactLog)
+		errors.RedactLogEnabled.Store(val)
 		return nil
 	}},
 	{Scope: ScopeGlobal | ScopeSession, Name: TiDBShardAllocateStep, Value: strconv.Itoa(DefTiDBShardAllocateStep), Type: TypeInt, MinValue: 1, MaxValue: uint64(math.MaxInt64), SetSession: func(s *SessionVars, val string) error {
diff --git a/pkg/store/driver/txn/driver_test.go b/pkg/store/driver/txn/driver_test.go
@@ -70,7 +70,7 @@ func TestWriteConflictPrettyFormat(t *testing.T) {
 
 	// test log redaction
 	original := errors.RedactLogEnabled.Load()
-	errors.RedactLogEnabled.Store(true)
+	errors.RedactLogEnabled.Store(errors.RedactLogEnable)
 	defer func() { errors.RedactLogEnabled.Store(original) }()
 	expectedStr = "[kv:9007]Write conflict, " +
 		"txnStartTS=399402937522847774, conflictStartTS=399402937719455772, conflictCommitTS=399402937719455773, " +
diff --git a/pkg/util/dbterror/terror_test.go b/pkg/util/dbterror/terror_test.go
@@ -29,7 +29,7 @@ func genErrMsg(pattern string, a ...any) string {
 
 func TestErrorRedact(t *testing.T) {
 	original := errors.RedactLogEnabled.Load()
-	errors.RedactLogEnabled.Store(true)
+	errors.RedactLogEnabled.Store(errors.RedactLogEnable)
 	defer func() { errors.RedactLogEnabled.Store(original) }()
 
 	class := ErrClass{}
diff --git a/pkg/util/logutil/consistency/BUILD.bazel b/pkg/util/logutil/consistency/BUILD.bazel
@@ -15,6 +15,7 @@ go_library(
         "//pkg/types",
         "//pkg/util/dbterror",
         "//pkg/util/logutil",
+        "//pkg/util/redact",
         "@com_github_pingcap_kvproto//pkg/kvrpcpb",
         "@com_github_tikv_client_go_v2//tikv",
         "@org_uber_go_zap//:zap",
diff --git a/pkg/util/logutil/consistency/reporter.go b/pkg/util/logutil/consistency/reporter.go
diff --git a/pkg/util/redact/redact.go b/pkg/util/redact/redact.go
diff --git a/pkg/util/redact/redact_test.go b/pkg/util/redact/redact_test.go

Original file line number	Diff line number	Diff line change
`@@ -1187,7 +1187,7 @@ func errStrForLog(err error, redactMode string) string {`
`1187`	`1187`	`} else {`
`1188`	`1188`	`ret = errors.ErrorStack(err)`
`1189`	`1189`	`}`
`1190`		`- return redact.Redact(redactMode, ret)`
	`1190`	`+ return redact.String(redactMode, ret)`
`1191`	`1191`	`}`
`1192`	`1192`
`1193`	`1193`	`func (cc *clientConn) addMetrics(cmd byte, startTime time.Time, err error) {`
Original file line number	Diff line number	Diff line change
`@@ -583,7 +583,7 @@ func (cc *clientConn) preparedStmt2String(stmtID uint32) string {`
`583`	`583`	`}`
`584`	`584`	`sql := parser.Normalize(cc.preparedStmt2StringNoArgs(stmtID), sv.EnableRedactNew)`
`585`	`585`	`if m := sv.EnableRedactNew; m != "ON" {`
`586`		`- sql += redact.Redact(sv.EnableRedactNew, sv.PlanCacheParams.String())`
	`586`	`+ sql += redact.String(sv.EnableRedactNew, sv.PlanCacheParams.String())`
`587`	`587`	`}`
`588`	`588`	`return sql`
`589`	`589`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ type statementBuildInfo struct {`
`69`	`69`	`}`
`70`	`70`
`71`	`71`	`func (j job) String(redacted string) string {`
`72`		`- return fmt.Sprintf("job id: %d, estimated size: %d, sql: %s", j.jobID, j.jobSize, redact.Redact(redacted, j.sql))`
	`72`	`+ return fmt.Sprintf("job id: %d, estimated size: %d, sql: %s", j.jobID, j.jobSize, redact.String(redacted, j.sql))`
`73`	`73`	`}`
`74`	`74`
`75`	`75`	`// HandleNonTransactionalDML is the entry point for a non-transactional DML statement`