This is an automated cherry-pick of pingcap#46138

Signed-off-by: ti-chi-bot <[email protected]>

Author: winoros

build/BUILD.bazel

@@ -150,5 +150,13 @@ nogo(
                    "//build/linter/revive",
                ],
                "//conditions:default": [],
+<<<<<<< HEAD
+=======
+           }) +
+           select({
+               "//build:without_rbe": [
+               ],
+               "//conditions:default": [],
+>>>>>>> 51f1a828e47 (statistics: record last gc ts to avoid huge read on stats table (#46138))
            }),
 )

ddl/partition.go

@@ -1959,6 +1959,13 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
 				return ver, errors.Trace(err)
 			}
 		}
+<<<<<<< HEAD
+=======
+		if tblInfo.TiFlashReplica != nil {
+			removeTiFlashAvailablePartitionIDs(tblInfo, physicalTableIDs)
+		}
+		droppedDefs := tblInfo.Partition.DroppingDefinitions
+>>>>>>> 51f1a828e47 (statistics: record last gc ts to avoid huge read on stats table (#46138))
 		tblInfo.Partition.DroppingDefinitions = nil
 		// used by ApplyDiff in updateSchemaVersion
 		job.CtxVars = []interface{}{physicalTableIDs}
@@ -1968,7 +1975,7 @@ func (w *worker) onDropTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (
 		}
 		job.SchemaState = model.StateNone
 		job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo)
-		asyncNotifyEvent(d, &util.Event{Tp: model.ActionDropTablePartition, TableInfo: tblInfo, PartInfo: &model.PartitionInfo{Definitions: tblInfo.Partition.Definitions}})
+		asyncNotifyEvent(d, &util.Event{Tp: model.ActionDropTablePartition, TableInfo: tblInfo, PartInfo: &model.PartitionInfo{Definitions: droppedDefs}})
 		// A background job will be created to delete old partition data.
 		job.Args = []interface{}{physicalTableIDs}
 	default:

ddl/table.go

@@ -393,6 +393,11 @@ func onDropTableOrView(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ er
 		job.FinishTableJob(model.JobStateDone, model.StateNone, ver, tblInfo)
 		startKey := tablecodec.EncodeTablePrefix(job.TableID)
 		job.Args = append(job.Args, startKey, oldIDs, ruleIDs)
+		if tblInfo.IsSequence() {
+			asyncNotifyEvent(d, &util.Event{Tp: model.ActionDropSequence, TableInfo: tblInfo})
+		} else if !tblInfo.IsView() {
+			asyncNotifyEvent(d, &util.Event{Tp: model.ActionDropTable, TableInfo: tblInfo})
+		}
 	default:
 		return ver, errors.Trace(dbterror.ErrInvalidDDLState.GenWithStackByArgs("table", tblInfo.State))
 	}

statistics/handle/ddl.go

@@ -41,6 +41,13 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
 				return err
 			}
 		}
+	case model.ActionDropTable:
+		ids := h.getInitStateTableIDs(t.TableInfo)
+		for _, id := range ids {
+			if err := h.resetTableStats2KVForDrop(id); err != nil {
+				return err
+			}
+		}
 	case model.ActionAddColumn, model.ActionModifyColumn:
 		ids := h.getInitStateTableIDs(t.TableInfo)
 		for _, id := range ids {
@@ -61,6 +68,11 @@ func (h *Handle) HandleDDLEvent(t *util.Event) error {
 				return err
 			}
 		}
+		for _, def := range t.PartInfo.Definitions {
+			if err := h.resetTableStats2KVForDrop(def.ID); err != nil {
+				return err
+			}
+		}
 	case model.ActionReorganizePartition:
 		for _, def := range t.PartInfo.Definitions {
 			// TODO: Should we trigger analyze instead of adding 0s?
@@ -266,6 +278,36 @@ func (h *Handle) insertTableStats2KV(info *model.TableInfo, physicalID int64) (e
 	return nil
 }
 
+// resetTableStats2KV resets the count to 0.
+func (h *Handle) resetTableStats2KVForDrop(physicalID int64) (err error) {
+	statsVer := uint64(0)
+	defer func() {
+		if err == nil && statsVer != 0 {
+			h.recordHistoricalStatsMeta(physicalID, statsVer, StatsMetaHistorySourceSchemaChange)
+		}
+	}()
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	ctx := kv.WithInternalSourceType(context.Background(), kv.InternalTxnStats)
+	exec := h.mu.ctx.(sqlexec.SQLExecutor)
+	_, err = exec.ExecuteInternal(ctx, "begin")
+	if err != nil {
+		return errors.Trace(err)
+	}
+	defer func() {
+		err = finishTransaction(ctx, exec, err)
+	}()
+	txn, err := h.mu.ctx.Txn(true)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	startTS := txn.StartTS()
+	if _, err := exec.ExecuteInternal(ctx, "update mysql.stats_meta set version=%? where table_id =%?", startTS, physicalID); err != nil {
+		return err
+	}
+	return nil
+}
+
 // insertColStats2KV insert a record to stats_histograms with distinct_count 1 and insert a bucket to stats_buckets with default value.
 // This operation also updates version.
 func (h *Handle) insertColStats2KV(physicalID int64, colInfos []*model.ColumnInfo) (err error) {

statistics/handle/ddl_test.go

@@ -213,11 +213,16 @@ func TestDDLHistogram(t *testing.T) {
 func TestDDLPartition(t *testing.T) {
 	store, do := testkit.CreateMockStoreAndDomain(t)
 	testKit := testkit.NewTestKit(t, store)
-	for _, pruneMode := range []string{"static", "dynamic"} {
+	for i, pruneMode := range []string{"static", "dynamic"} {
 		testKit.MustExec("set @@tidb_partition_prune_mode=`" + pruneMode + "`")
 		testKit.MustExec("set global tidb_partition_prune_mode=`" + pruneMode + "`")
 		testKit.MustExec("use test")
 		testKit.MustExec("drop table if exists t")
+		h := do.StatsHandle()
+		if i == 1 {
+			err := h.HandleDDLEvent(<-h.DDLEventCh())
+			require.NoError(t, err)
+		}
 		createTable := `CREATE TABLE t (a int, b int, primary key(a), index idx(b))
 PARTITION BY RANGE ( a ) (
 		PARTITION p0 VALUES LESS THAN (6),
@@ -230,14 +235,13 @@ PARTITION BY RANGE ( a ) (
 		tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
 		require.NoError(t, err)
 		tableInfo := tbl.Meta()
-		h := do.StatsHandle()
 		err = h.HandleDDLEvent(<-h.DDLEventCh())
 		require.NoError(t, err)
 		require.Nil(t, h.Update(is))
 		pi := tableInfo.GetPartitionInfo()
 		for _, def := range pi.Definitions {
 			statsTbl := h.GetPartitionStats(tableInfo, def.ID)
-			require.False(t, statsTbl.Pseudo)
+			require.False(t, statsTbl.Pseudo, "for %v", pruneMode)
 		}
 
 		testKit.MustExec("insert into t values (1,2),(6,2),(11,2),(16,2)")

statistics/handle/gc.go

@@ -17,6 +17,7 @@ package handle
 import (
 	"context"
 	"encoding/json"
+	"strconv"
 	"time"
 
 	"github.com/pingcap/errors"
@@ -33,9 +34,11 @@ import (
 	"go.uber.org/zap"
 )
 
+const gcLastTSVarName = "tidb_stats_gc_last_ts"
+
 // GCStats will garbage collect the useless stats info. For dropped tables, we will first update their version so that
 // other tidb could know that table is deleted.
-func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error {
+func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) (err error) {
 	ctx := context.Background()
 	// To make sure that all the deleted tables' schema and stats info have been acknowledged to all tidb,
 	// we only garbage collect version before 10 lease.
@@ -46,7 +49,17 @@ func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error
 		return nil
 	}
 	gcVer := now - offset
-	rows, _, err := h.execRestrictedSQL(ctx, "select table_id from mysql.stats_meta where version < %?", gcVer)
+	lastGC, err := h.GetLastGCTimestamp(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			return
+		}
+		err = h.writeGCTimestampToKV(ctx, gcVer)
+	}()
+	rows, _, err := h.execRestrictedSQL(ctx, "select table_id from mysql.stats_meta where version >= %? and version < %?", lastGC, gcVer)
 	if err != nil {
 		return errors.Trace(err)
 	}
@@ -69,6 +82,33 @@ func (h *Handle) GCStats(is infoschema.InfoSchema, ddlLease time.Duration) error
 	return h.removeDeletedExtendedStats(gcVer)
 }
 
+// GetLastGCTimestamp loads the last gc time from mysql.tidb.
+func (h *Handle) GetLastGCTimestamp(ctx context.Context) (uint64, error) {
+	rows, _, err := h.execRestrictedSQL(ctx, "SELECT HIGH_PRIORITY variable_value FROM mysql.tidb WHERE variable_name=%?", gcLastTSVarName)
+	if err != nil {
+		return 0, errors.Trace(err)
+	}
+	if len(rows) == 0 {
+		return 0, nil
+	}
+	lastGcTSString := rows[0].GetString(0)
+	lastGcTS, err := strconv.ParseUint(lastGcTSString, 10, 64)
+	if err != nil {
+		return 0, errors.Trace(err)
+	}
+	return lastGcTS, nil
+}
+
+func (h *Handle) writeGCTimestampToKV(ctx context.Context, newTS uint64) error {
+	_, _, err := h.execRestrictedSQL(ctx,
+		"insert into mysql.tidb (variable_name, variable_value) values (%?, %?) on duplicate key update variable_value = %?",
+		gcLastTSVarName,
+		newTS,
+		newTS,
+	)
+	return err
+}
+
 func (h *Handle) gcTableStats(is infoschema.InfoSchema, physicalID int64) error {
 	ctx := context.Background()
 	rows, _, err := h.execRestrictedSQL(ctx, "select is_index, hist_id from mysql.stats_histograms where table_id = %?", physicalID)

statistics/handle/gc_test.go

@@ -103,6 +103,8 @@ func TestGCExtendedStats(t *testing.T) {
 	testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3)")
 	testKit.MustExec("alter table t add stats_extended s1 correlation(a,b)")
 	testKit.MustExec("alter table t add stats_extended s2 correlation(b,c)")
+	h := dom.StatsHandle()
+	require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
 	testKit.MustExec("analyze table t")
 
 	testKit.MustQuery("select name, type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows(
@@ -114,7 +116,6 @@ func TestGCExtendedStats(t *testing.T) {
 		"s1 2 [1,2] 1.000000 1",
 		"s2 2 [2,3] 1.000000 1",
 	))
-	h := dom.StatsHandle()
 	ddlLease := time.Duration(0)
 	require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease))
 	testKit.MustQuery("select name, type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows(
@@ -130,6 +131,7 @@ func TestGCExtendedStats(t *testing.T) {
 	testKit.MustQuery("select name, type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows(
 		"s2 2 [2,3] 1.000000 1",
 	))
+	require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
 	require.Nil(t, h.GCStats(dom.InfoSchema(), ddlLease))
 	testKit.MustQuery("select name, type, column_ids, stats, status from mysql.stats_extended").Sort().Check(testkit.Rows(
 		"s2 2 [2,3] 1.000000 2",

statistics/integration_test.go

@@ -913,6 +913,7 @@ func TestIssue49986(t *testing.T) {
 	tk := testkit.NewTestKit(t, store)
 	tk.MustExec("use test")
 
+<<<<<<< HEAD
 	tk.MustExec("drop table if exists test.t;")
 	tk.MustExec("create table if not exists test.ast (i varchar(20));")
 	tk.MustExec("create table if not exists test.acc (j varchar(20), k varchar(20), l varchar(20), m varchar(20));")
@@ -927,4 +928,69 @@ func TestIssue49986(t *testing.T) {
 			"      └─TableReader(Probe) 10.00 root  data:Selection",
 			"        └─Selection 10.00 cop[tikv]  eq(\"astp2019121731703151\", test.acc.m)",
 			"          └─TableFullScan 10000.00 cop[tikv] table:b keep order:false, stats:pseudo"))
+=======
+	for i, version := range []string{"1", "2"} {
+		tk.MustExec("set @@session.tidb_analyze_version = " + version)
+
+		// analyze table t
+		tk.MustExec("drop table if exists t")
+		if i != 0 {
+			require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		}
+		tk.MustExec("CREATE TABLE t ( a int, b int, c int default 0, key(a) )" +
+			"PARTITION BY RANGE (a) (" +
+			"PARTITION p0 VALUES LESS THAN (10)," +
+			"PARTITION p1 VALUES LESS THAN (20)," +
+			"PARTITION p2 VALUES LESS THAN (30)," +
+			"PARTITION p3 VALUES LESS THAN (40))")
+		require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		tk.MustExec("insert into t(a,b) values (1,1), (2,2), (3,3), (15,15), (25,25), (35,35)")
+		tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b)")
+		require.Nil(t, h.DumpStatsDeltaToKV(handle.DumpAll))
+		tk.MustExec("analyze table t")
+		require.Nil(t, h.Update(dom.InfoSchema()))
+		tk.MustQuery("SELECT b FROM t use index(idx) WHERE b < 16 ORDER BY b").
+			Check(testkit.Rows("1", "2", "3", "15"))
+		tk.MustQuery("EXPLAIN SELECT b FROM t use index(idx) WHERE b < 16 ORDER BY b").
+			Check(testkit.Rows("IndexReader_12 4.00 root partition:all index:IndexRangeScan_11",
+				"└─IndexRangeScan_11 4.00 cop[tikv] table:t, index:idx(b) range:[-inf,16), keep order:true"))
+
+		// analyze table t index idx
+		tk.MustExec("drop table if exists t")
+		require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		tk.MustExec("CREATE TABLE t ( a int, b int, c int default 0, primary key(b, a) clustered )" +
+			"PARTITION BY RANGE (a) (" +
+			"PARTITION p0 VALUES LESS THAN (10)," +
+			"PARTITION p1 VALUES LESS THAN (20)," +
+			"PARTITION p2 VALUES LESS THAN (30)," +
+			"PARTITION p3 VALUES LESS THAN (40));")
+		require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		tk.MustExec("insert into t(a,b) values (1,1), (2,2), (3,3), (15,15), (25,25), (35,35)")
+		tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b);")
+		require.Nil(t, h.DumpStatsDeltaToKV(handle.DumpAll))
+		tk.MustExec("analyze table t index idx")
+		require.Nil(t, h.Update(dom.InfoSchema()))
+		rows := tk.MustQuery("EXPLAIN SELECT b FROM t use index(idx) WHERE b < 16 ORDER BY b;").Rows()
+		require.Equal(t, "4.00", rows[0][1])
+
+		// analyze table t index
+		tk.MustExec("drop table if exists t")
+		require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		tk.MustExec("CREATE TABLE t ( a int, b int, c int default 0, primary key(b, a) clustered )" +
+			"PARTITION BY RANGE (a) (" +
+			"PARTITION p0 VALUES LESS THAN (10)," +
+			"PARTITION p1 VALUES LESS THAN (20)," +
+			"PARTITION p2 VALUES LESS THAN (30)," +
+			"PARTITION p3 VALUES LESS THAN (40));")
+		require.Nil(t, h.HandleDDLEvent(<-h.DDLEventCh()))
+		tk.MustExec("insert into t(a,b) values (1,1), (2,2), (3,3), (15,15), (25,25), (35,35)")
+		tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b);")
+		require.Nil(t, h.DumpStatsDeltaToKV(handle.DumpAll))
+		tk.MustExec("analyze table t index")
+		require.Nil(t, h.Update(dom.InfoSchema()))
+		tk.MustQuery("EXPLAIN SELECT b FROM t use index(idx) WHERE b < 16 ORDER BY b;").
+			Check(testkit.Rows("IndexReader_12 4.00 root partition:all index:IndexRangeScan_11",
+				"└─IndexRangeScan_11 4.00 cop[tikv] table:t, index:idx(b) range:[-inf,16), keep order:true"))
+	}
+>>>>>>> 51f1a828e47 (statistics: record last gc ts to avoid huge read on stats table (#46138))
 }