Skip to content

Commit e2696ed

Browse files
ti-chi-botti-chi-bot
authored
stats, infoschema: avoid some network cost of reading table meta kv (pingcap#59105) (pingcap#59594)
ref pingcap#57869, ref pingcap#59104
1 parent a546bc5 commit e2696ed

File tree

9 files changed

+122
-35
lines changed

9 files changed

+122
-35
lines changed

‎pkg/infoschema/infoschema.go

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -315,17 +315,17 @@ func (is *infoSchema) TableByID(_ stdctx.Context, id int64) (val table.Table, ok
315315
return slice[idx], true
316316
}
317317

318-
func (is *infoSchema) SchemaNameByTableID(tableID int64) (schemaName pmodel.CIStr, ok bool) {
319-
tbl, ok := is.TableByID(stdctx.Background(), tableID)
318+
// TableItemByID implements InfoSchema.TableItemByID.
319+
func (is *infoSchema) TableItemByID(id int64) (TableItem, bool) {
320+
tbl, ok := is.TableByID(stdctx.Background(), id)
320321
if !ok {
321-
return
322+
return TableItem{}, false
322323
}
323324
db, ok := is.SchemaByID(tbl.Meta().DBID)
324325
if !ok {
325-
return
326+
return TableItem{}, false
326327
}
327-
328-
return db.Name, true
328+
return TableItem{DBName: db.Name, TableName: tbl.Meta().Name}, true
329329
}
330330

331331
// TableInfoByID implements InfoSchema.TableInfoByID
@@ -412,6 +412,23 @@ func (is *infoSchema) AllSchemaNames() (schemas []pmodel.CIStr) {
412412
return rs
413413
}
414414

415+
func (is *infoSchema) TableItemByPartitionID(partitionID int64) (TableItem, bool) {
416+
tbl, db, _ := is.FindTableByPartitionID(partitionID)
417+
if tbl == nil {
418+
return TableItem{}, false
419+
}
420+
return TableItem{DBName: db.Name, TableName: tbl.Meta().Name}, true
421+
}
422+
423+
// TableIDByPartitionID implements InfoSchema.TableIDByPartitionID.
424+
func (is *infoSchema) TableIDByPartitionID(partitionID int64) (tableID int64, ok bool) {
425+
tbl, _, _ := is.FindTableByPartitionID(partitionID)
426+
if tbl == nil {
427+
return
428+
}
429+
return tbl.Meta().ID, true
430+
}
431+
415432
// FindTableByPartitionID finds the partition-table info by the partitionID.
416433
// FindTableByPartitionID will traverse all the tables to find the partitionID partition in which partition-table.
417434
func (is *infoSchema) FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition) {

‎pkg/infoschema/infoschema_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,9 +283,9 @@ func TestBasic(t *testing.T) {
283283

284284
for _, tt := range tests {
285285
t.Run(tt.name, func(t *testing.T) {
286-
gotSchema, gotOK := is.SchemaNameByTableID(tt.tableID)
286+
gotItem, gotOK := is.TableItemByID(tt.tableID)
287287
require.Equal(t, tt.wantOK, gotOK)
288-
require.Equal(t, tt.wantSchema, gotSchema)
288+
require.Equal(t, tt.wantSchema, gotItem.DBName)
289289
})
290290
}
291291
}

‎pkg/infoschema/infoschema_v2.go

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -651,17 +651,17 @@ func (is *infoschemaV2) TableByID(ctx context.Context, id int64) (val table.Tabl
651651
return ret, true
652652
}
653653

654-
func (is *infoschemaV2) SchemaNameByTableID(tableID int64) (schemaName pmodel.CIStr, ok bool) {
654+
// TableItemByID implements the InfoSchema interface.
655+
// It only contains memory operations, no worries about accessing the storage.
656+
func (is *infoschemaV2) TableItemByID(tableID int64) (TableItem, bool) {
655657
if !tableIDIsValid(tableID) {
656-
return
658+
return TableItem{}, false
657659
}
658-
659660
itm, ok := is.searchTableItemByID(tableID)
660661
if !ok {
661-
return
662+
return TableItem{}, false
662663
}
663-
664-
return itm.dbName, true
664+
return TableItem{DBName: itm.dbName, TableName: itm.tableName}, true
665665
}
666666

667667
// TableItem is exported from tableItem.
@@ -1023,9 +1023,7 @@ func (is *infoschemaV2) SchemaExists(schema pmodel.CIStr) bool {
10231023
return ok
10241024
}
10251025

1026-
func (is *infoschemaV2) FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition) {
1027-
var ok bool
1028-
var pi partitionItem
1026+
func (is *infoschemaV2) searchPartitionItemByPartitionID(partitionID int64) (pi partitionItem, ok bool) {
10291027
is.pid2tid.Descend(partitionItem{partitionID: partitionID, schemaVersion: math.MaxInt64},
10301028
func(item partitionItem) bool {
10311029
if item.partitionID != partitionID {
@@ -1036,12 +1034,37 @@ func (is *infoschemaV2) FindTableByPartitionID(partitionID int64) (table.Table,
10361034
return true
10371035
}
10381036
if item.schemaVersion <= is.infoSchema.schemaMetaVersion {
1039-
ok = !item.tomb
10401037
pi = item
1038+
ok = !item.tomb
10411039
return false
10421040
}
10431041
return true
1044-
})
1042+
},
1043+
)
1044+
return pi, ok
1045+
}
1046+
1047+
// TableItemByPartitionID implements InfoSchema.TableItemByPartitionID.
1048+
// It returns the lightweight meta info, no worries about access the storage.
1049+
func (is *infoschemaV2) TableItemByPartitionID(partitionID int64) (TableItem, bool) {
1050+
pi, ok := is.searchPartitionItemByPartitionID(partitionID)
1051+
if !ok {
1052+
return TableItem{}, false
1053+
}
1054+
return is.TableItemByID(pi.tableID)
1055+
}
1056+
1057+
// TableIDByPartitionID implements InfoSchema.TableIDByPartitionID.
1058+
func (is *infoschemaV2) TableIDByPartitionID(partitionID int64) (tableID int64, ok bool) {
1059+
pi, ok := is.searchPartitionItemByPartitionID(partitionID)
1060+
if !ok {
1061+
return
1062+
}
1063+
return pi.tableID, true
1064+
}
1065+
1066+
func (is *infoschemaV2) FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition) {
1067+
pi, ok := is.searchPartitionItemByPartitionID(partitionID)
10451068
if !ok {
10461069
return nil, nil, nil
10471070
}

‎pkg/infoschema/infoschema_v2_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,10 @@ func TestV2Basic(t *testing.T) {
152152

153153
for _, tt := range schemaNameByTableIDTests {
154154
t.Run(tt.name, func(t *testing.T) {
155-
gotSchema, gotOK := is.SchemaNameByTableID(tt.tableID)
155+
gotItem, gotOK := is.TableItemByID(tt.tableID)
156156

157157
require.Equal(t, tt.wantOK, gotOK)
158-
require.Equal(t, tt.wantSchema, gotSchema)
158+
require.Equal(t, tt.wantSchema, gotItem.DBName)
159159
})
160160
}
161161

‎pkg/infoschema/interface.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,17 @@ type InfoSchema interface {
3030
context.MetaOnlyInfoSchema
3131
TableByName(ctx stdctx.Context, schema, table pmodel.CIStr) (table.Table, error)
3232
TableByID(ctx stdctx.Context, id int64) (table.Table, bool)
33-
SchemaNameByTableID(tableID int64) (pmodel.CIStr, bool)
33+
// TableItemByID returns a lightweight table meta specified by the given ID,
34+
// without loading the whole info from storage.
35+
// So it's all in memory operation. No need to worry about network or disk cost.
36+
TableItemByID(id int64) (TableItem, bool)
37+
// TableIDByPartitionID is a pure memory operation, returns the table ID by the partition ID.
38+
// It's all in memory operation. No need to worry about network or disk cost.
39+
TableIDByPartitionID(partitionID int64) (tableID int64, ok bool)
3440
FindTableByPartitionID(partitionID int64) (table.Table, *model.DBInfo, *model.PartitionDefinition)
41+
// TableItemByPartitionID returns a lightweight table meta specified by the partition ID,
42+
// without loading the whole info from storage.
43+
// So it's all in memory operation. No need to worry about network or disk cost.
44+
TableItemByPartitionID(partitionID int64) (TableItem, bool)
3545
base() *infoSchema
3646
}

‎pkg/statistics/handle/handle.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,33 @@ func (h *Handle) getPartitionStats(tblInfo *model.TableInfo, pid int64, returnPs
205205
return tbl
206206
}
207207

208+
// GetPartitionStatsByID retrieves the partition stats from cache by partition ID.
209+
func (h *Handle) GetPartitionStatsByID(is infoschema.InfoSchema, pid int64) *statistics.Table {
210+
return h.getPartitionStatsByID(is, pid)
211+
}
212+
213+
func (h *Handle) getPartitionStatsByID(is infoschema.InfoSchema, pid int64) *statistics.Table {
214+
var statsTbl *statistics.Table
215+
intest.Assert(h != nil, "stats handle is nil")
216+
tbl, ok := h.Get(pid)
217+
if !ok {
218+
tbl, ok := h.TableInfoByID(is, pid)
219+
if !ok {
220+
return nil
221+
}
222+
// TODO: it's possible don't rely on the full table meta to do it here.
223+
statsTbl = statistics.PseudoTable(tbl.Meta(), false, true)
224+
statsTbl.PhysicalID = pid
225+
if tbl.Meta().GetPartitionInfo() == nil || h.Len() < 64 {
226+
h.UpdateStatsCache(types.CacheUpdate{
227+
Updated: []*statistics.Table{statsTbl},
228+
})
229+
}
230+
return nil
231+
}
232+
return tbl
233+
}
234+
208235
// FlushStats flushes the cached stats update into store.
209236
func (h *Handle) FlushStats() {
210237
if err := h.DumpStatsDeltaToKV(true); err != nil {

‎pkg/statistics/handle/types/interfaces.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,9 @@ type StatsHandle interface {
546546
// GetPartitionStats retrieves the partition stats from cache.
547547
GetPartitionStats(tblInfo *model.TableInfo, pid int64) *statistics.Table
548548

549+
// GetPartitionStatsByID retrieves the partition stats from cache by partition ID.
550+
GetPartitionStatsByID(is infoschema.InfoSchema, pid int64) *statistics.Table
551+
549552
// GetPartitionStatsForAutoAnalyze retrieves the partition stats from cache, but it will not return pseudo.
550553
GetPartitionStatsForAutoAnalyze(tblInfo *model.TableInfo, pid int64) *statistics.Table
551554

‎pkg/statistics/handle/usage/session_stats_collect.go

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,11 @@ var (
5555
// 3. If the stats delta haven't been dumped in the past hour, then return true.
5656
// 4. If the table stats is pseudo or empty or `Modify Count / Table Count` exceeds the threshold.
5757
func (s *statsUsageImpl) needDumpStatsDelta(is infoschema.InfoSchema, dumpAll bool, id int64, item variable.TableDelta, currentTime time.Time) bool {
58-
tbl, ok := s.statsHandle.TableInfoByID(is, id)
58+
tableItem, ok := s.statsHandle.TableItemByID(is, id)
5959
if !ok {
6060
return false
6161
}
62-
dbInfo, ok := infoschema.SchemaByTable(is, tbl.Meta())
63-
if !ok {
64-
return false
65-
}
66-
if util.IsMemOrSysDB(dbInfo.Name.L) {
62+
if util.IsMemOrSysDB(tableItem.DBName.L) {
6763
return false
6864
}
6965
if dumpAll {
@@ -76,8 +72,8 @@ func (s *statsUsageImpl) needDumpStatsDelta(is infoschema.InfoSchema, dumpAll bo
7672
// Dump the stats to kv at least once 5 minutes.
7773
return true
7874
}
79-
statsTbl := s.statsHandle.GetPartitionStats(tbl.Meta(), id)
80-
if statsTbl.Pseudo || statsTbl.RealtimeCount == 0 || float64(item.Count)/float64(statsTbl.RealtimeCount) > DumpStatsDeltaRatio {
75+
statsTbl := s.statsHandle.GetPartitionStatsByID(is, id)
76+
if statsTbl == nil || statsTbl.Pseudo || statsTbl.RealtimeCount == 0 || float64(item.Count)/float64(statsTbl.RealtimeCount) > DumpStatsDeltaRatio {
8177
// Dump the stats when there are many modifications.
8278
return true
8379
}
@@ -242,8 +238,8 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV(
242238
// Add psychical table ID.
243239
allTableIDs = append(allTableIDs, update.TableID)
244240
// Add parent table ID if it's a partition table.
245-
if tbl, _, _ := is.FindTableByPartitionID(update.TableID); tbl != nil {
246-
allTableIDs = append(allTableIDs, tbl.Meta().ID)
241+
if tblID, ok := is.TableIDByPartitionID(update.TableID); ok {
242+
allTableIDs = append(allTableIDs, tblID)
247243
}
248244
}
249245

@@ -260,9 +256,8 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV(
260256
continue
261257
}
262258

263-
tbl, _, _ := is.FindTableByPartitionID(update.TableID)
264-
if tbl != nil { // It's a partition table.
265-
tableID := tbl.Meta().ID
259+
tableID, ok := is.TableIDByPartitionID(update.TableID)
260+
if ok { // It's a partition table.
266261
isTableLocked := false
267262
isPartitionLocked := false
268263

‎pkg/statistics/handle/util/table_info.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ type TableInfoGetter interface {
2626
// TableInfoByID returns the table info specified by the physicalID.
2727
// If the physicalID is corresponding to a partition, return its parent table.
2828
TableInfoByID(is infoschema.InfoSchema, physicalID int64) (table.Table, bool)
29+
// TableItemByID returns the schema name and table name specified by the physicalID.
30+
// This is pure memory operation.
31+
TableItemByID(is infoschema.InfoSchema, id int64) (infoschema.TableItem, bool)
2932
}
3033

3134
// tableInfoGetterImpl is used to get table meta info.
@@ -47,3 +50,12 @@ func (*tableInfoGetterImpl) TableInfoByID(is infoschema.InfoSchema, physicalID i
4750
tbl, _, _ = is.FindTableByPartitionID(physicalID)
4851
return tbl, tbl != nil
4952
}
53+
54+
// TableItemByID returns the lightweight table meta specified by the physicalID.
55+
func (*tableInfoGetterImpl) TableItemByID(is infoschema.InfoSchema, id int64) (infoschema.TableItem, bool) {
56+
tableItem, ok := is.TableItemByID(id)
57+
if ok {
58+
return tableItem, true
59+
}
60+
return is.TableItemByPartitionID(id)
61+
}

0 commit comments

Comments
 (0)