Skip to content

Commit 281462c

Browse files
qw4990qw4990
authored
planner: support recommend index set <opt>=<val> (#56249)
ref #12303
1 parent 19caf52 commit 281462c

File tree

7 files changed

+385
-38
lines changed

7 files changed

+385
-38
lines changed

‎pkg/executor/executor.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2744,14 +2744,15 @@ func (e *RecommendIndexExec) Next(ctx context.Context, req *chunk.Chunk) error {
27442744
}
27452745
e.done = true
27462746

2747+
if e.Action == "set" {
2748+
return indexadvisor.SetOption(e.Ctx(), e.Option, e.Value)
2749+
}
2750+
27472751
if e.Action != "run" {
27482752
return fmt.Errorf("unsupported action: %s", e.Action)
27492753
}
27502754

2751-
opt := &indexadvisor.Option{
2752-
MaxNumIndexes: 3,
2753-
MaxIndexWidth: 3,
2754-
}
2755+
opt := &indexadvisor.Option{}
27552756
if e.SQL != "" {
27562757
opt.SpecifiedSQLs = []string{e.SQL}
27572758
}

‎pkg/planner/core/planbuilder.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5740,6 +5740,11 @@ func (*PlanBuilder) buildRecommendIndex(v *ast.RecommendIndexStmt) (base.Plan, e
57405740
schema.Append(buildColumnWithName("", "reason", mysql.TypeVarchar, 256))
57415741
schema.Append(buildColumnWithName("", "top_impacted_query", mysql.TypeBlob, -1))
57425742
p.setSchemaAndNames(schema.col2Schema(), schema.names)
5743+
case "set":
5744+
p.Option = strings.TrimSpace(p.Option)
5745+
if p.Option == "" {
5746+
return nil, fmt.Errorf("option is empty")
5747+
}
57435748
default:
57445749
return nil, fmt.Errorf("unsupported action %s", v.Action)
57455750
}

‎pkg/planner/indexadvisor/BUILD.bazel‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
"indexadvisor.go",
88
"model.go",
99
"optimizer.go",
10+
"options.go",
1011
"utils.go",
1112
],
1213
importpath = "github.com/pingcap/tidb/pkg/planner/indexadvisor",
@@ -32,6 +33,7 @@ go_library(
3233
"//pkg/util/set",
3334
"//pkg/util/sqlexec",
3435
"@com_github_google_uuid//:uuid",
36+
"@com_github_pkg_errors//:errors",
3537
"@org_uber_go_zap//:zap",
3638
],
3739
)
@@ -44,10 +46,11 @@ go_test(
4446
"indexadvisor_test.go",
4547
"indexadvisor_tpch_test.go",
4648
"optimizer_test.go",
49+
"options_test.go",
4750
"utils_test.go",
4851
],
4952
flaky = True,
50-
shard_count = 34,
53+
shard_count = 38,
5154
deps = [
5255
":indexadvisor",
5356
"//pkg/parser/mysql",

‎pkg/planner/indexadvisor/algorithm.go‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package indexadvisor
1717
import (
1818
"fmt"
1919
"strings"
20+
"time"
2021

2122
"github.com/google/uuid"
2223
s "github.com/pingcap/tidb/pkg/util/set"
@@ -33,11 +34,11 @@ import (
3334

3435
// adviseIndexes implements the auto-admin algorithm.
3536
func adviseIndexes(querySet s.Set[Query], indexableColSet s.Set[Column],
36-
maxIndexes, maxIndexWidth int, optimizer Optimizer) (s.Set[Index], error) {
37+
optimizer Optimizer, option *Option) (s.Set[Index], error) {
3738
aa := &autoAdmin{
38-
optimizer: optimizer,
39-
maxIndexes: maxIndexes,
40-
maxIndexWidth: maxIndexWidth,
39+
optimizer: optimizer,
40+
option: option,
41+
startAt: time.Now(),
4142
}
4243

4344
bestIndexes, err := aa.calculateBestIndexes(querySet, indexableColSet)
@@ -48,40 +49,46 @@ func adviseIndexes(querySet s.Set[Query], indexableColSet s.Set[Column],
4849
}
4950

5051
type autoAdmin struct {
51-
optimizer Optimizer
52-
maxIndexes int // The algorithm stops as soon as it has selected #max_indexes indexes
53-
maxIndexWidth int // The number of columns an index can contain at maximum.
52+
optimizer Optimizer
53+
option *Option
54+
startAt time.Time
5455
}
5556

5657
func (aa *autoAdmin) calculateBestIndexes(querySet s.Set[Query], indexableColSet s.Set[Column]) (s.Set[Index], error) {
57-
if aa.maxIndexes == 0 {
58+
if aa.option.MaxNumIndexes == 0 {
5859
return nil, nil
5960
}
6061

6162
potentialIndexes := s.NewSet[Index]() // each indexable column as a single-column index
6263
for _, col := range indexableColSet.ToList() {
6364
potentialIndexes.Add(NewIndex(col.SchemaName, col.TableName, aa.tempIndexName(col), col.ColumnName))
6465
}
66+
if err := aa.timeout(); err != nil {
67+
return nil, err
68+
}
6569

6670
currentBestIndexes := s.NewSet[Index]()
67-
for currentMaxIndexWidth := 1; currentMaxIndexWidth <= aa.maxIndexWidth; currentMaxIndexWidth++ {
71+
for currentMaxIndexWidth := 1; currentMaxIndexWidth <= aa.option.MaxIndexWidth; currentMaxIndexWidth++ {
6872
candidates, err := aa.selectIndexCandidates(querySet, potentialIndexes)
6973
if err != nil {
7074
return nil, err
7175
}
7276

73-
//maxIndexes := aa.maxIndexes * (aa.maxIndexWidth - currentMaxIndexWidth + 1)
74-
maxIndexes := aa.maxIndexes
77+
//maxIndexes := aa.option.MaxNumIndexes * (aa.option.MaxIndexWidth - currentMaxIndexWidth + 1)
78+
maxIndexes := aa.option.MaxNumIndexes
7579
currentBestIndexes, err = aa.enumerateCombinations(querySet, candidates, maxIndexes)
7680
if err != nil {
7781
return nil, err
7882
}
7983

80-
if currentMaxIndexWidth < aa.maxIndexWidth {
84+
if currentMaxIndexWidth < aa.option.MaxIndexWidth {
8185
// Update potential indexes for the next iteration
8286
potentialIndexes = currentBestIndexes
8387
potentialIndexes.Add(aa.createMultiColumnIndexes(indexableColSet, currentBestIndexes).ToList()...)
8488
}
89+
if err := aa.timeout(); err != nil {
90+
return nil, err
91+
}
8592
}
8693

8794
currentBestIndexes, err := aa.heuristicMergeIndexes(currentBestIndexes, querySet)
@@ -98,20 +105,24 @@ func (aa *autoAdmin) calculateBestIndexes(querySet s.Set[Query], indexableColSet
98105
return nil, err
99106
}
100107

101-
currentBestIndexes, err = aa.cutDown(currentBestIndexes, querySet, aa.optimizer, aa.maxIndexes)
108+
currentBestIndexes, err = aa.cutDown(currentBestIndexes, querySet, aa.optimizer, aa.option.MaxNumIndexes)
102109
if err != nil {
103110
return nil, err
104111
}
105112

113+
if err := aa.timeout(); err != nil {
114+
return nil, err
115+
}
116+
106117
// try to add more indexes if the number of indexes is less than maxIndexes
107-
for limit := 0; limit < 3 && currentBestIndexes.Size() < aa.maxIndexes; limit++ {
118+
for limit := 0; limit < 3 && currentBestIndexes.Size() < aa.option.MaxNumIndexes; limit++ {
108119
potentialIndexes = s.DiffSet(potentialIndexes, currentBestIndexes)
109120
currentCost, err := evaluateIndexSetCost(querySet, aa.optimizer, currentBestIndexes)
110121
if err != nil {
111122
return nil, err
112123
}
113124
currentBestIndexes, _, err = aa.enumerateGreedy(querySet, currentBestIndexes,
114-
currentCost, potentialIndexes, aa.maxIndexes)
125+
currentCost, potentialIndexes, aa.option.MaxNumIndexes)
115126
if err != nil {
116127
return nil, err
117128
}
@@ -120,6 +131,9 @@ func (aa *autoAdmin) calculateBestIndexes(querySet s.Set[Query], indexableColSet
120131
if err != nil {
121132
return nil, err
122133
}
134+
if err := aa.timeout(); err != nil {
135+
return nil, err
136+
}
123137
}
124138

125139
return currentBestIndexes, nil
@@ -167,7 +181,7 @@ func (aa *autoAdmin) heuristicCoveredIndexes(
167181
if err != nil {
168182
return nil, err
169183
}
170-
if selectCols == nil || selectCols.Size() == 0 || selectCols.Size() > aa.maxIndexWidth {
184+
if selectCols == nil || selectCols.Size() == 0 || selectCols.Size() > aa.option.MaxIndexWidth {
171185
continue
172186
}
173187
schemaName, tableName := selectCols.ToList()[0].SchemaName, selectCols.ToList()[0].TableName
@@ -178,7 +192,7 @@ func (aa *autoAdmin) heuristicCoveredIndexes(
178192
if idx.SchemaName != schemaName || idx.TableName != tableName {
179193
continue // not for the same table
180194
}
181-
if len(idx.Columns)+selectCols.Size() > aa.maxIndexWidth {
195+
if len(idx.Columns)+selectCols.Size() > aa.option.MaxIndexWidth {
182196
continue // exceed the max-index-width limitation
183197
}
184198
// try this cover-index: idx-cols + select-cols
@@ -279,8 +293,8 @@ func (aa *autoAdmin) heuristicMergeIndexes(
279293
}
280294
cols := []Column{col}
281295
cols = append(cols, orderByCols...)
282-
if len(cols) > aa.maxIndexWidth {
283-
cols = cols[:aa.maxIndexWidth]
296+
if len(cols) > aa.option.MaxIndexWidth {
297+
cols = cols[:aa.option.MaxIndexWidth]
284298
}
285299
idx = NewIndexWithColumns(aa.tempIndexName(cols...), cols...)
286300
contained = false
@@ -533,3 +547,10 @@ func (*autoAdmin) tempIndexName(cols ...Column) string {
533547

534548
return fmt.Sprintf("idx_%v", uuid.New().String())
535549
}
550+
551+
func (aa *autoAdmin) timeout() error {
552+
if time.Since(aa.startAt) > aa.option.Timeout {
553+
return fmt.Errorf("index advisor timeout after %v", aa.option.Timeout)
554+
}
555+
return nil
556+
}

‎pkg/planner/indexadvisor/indexadvisor.go‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"math"
2323
"sort"
2424
"strings"
25+
"time"
2526

2627
"github.com/pingcap/tidb/pkg/sessionctx"
2728
"github.com/pingcap/tidb/pkg/util/intest"
@@ -38,6 +39,8 @@ func TestKey(key string) string {
3839
type Option struct {
3940
MaxNumIndexes int
4041
MaxIndexWidth int
42+
MaxNumQuery int
43+
Timeout time.Duration
4144
SpecifiedSQLs []string
4245
}
4346

@@ -48,7 +51,12 @@ func AdviseIndexes(ctx context.Context, sctx sessionctx.Context,
4851
return nil, errors.New("nil input")
4952
}
5053

51-
advisorLogger().Info("start to recommend indexes")
54+
advisorLogger().Info("fill index advisor option")
55+
if err := fillOption(sctx, option); err != nil {
56+
advisorLogger().Error("fill index advisor option failed", zap.Error(err))
57+
return nil, err
58+
}
59+
advisorLogger().Info("index advisor option filled and start", zap.Any("option", option))
5260
defer func() {
5361
if r := recover(); r != nil {
5462
advisorLogger().Error("panic in AdviseIndexes", zap.Any("recover", r))
@@ -61,7 +69,7 @@ func AdviseIndexes(ctx context.Context, sctx sessionctx.Context,
6169
advisorLogger().Info("what-if optimizer prepared")
6270

6371
defaultDB := sctx.GetSessionVars().CurrentDB
64-
querySet, err := prepareQuerySet(ctx, sctx, defaultDB, opt, option.SpecifiedSQLs)
72+
querySet, err := prepareQuerySet(ctx, sctx, defaultDB, opt, option)
6573
if err != nil {
6674
advisorLogger().Error("prepare workload failed", zap.Error(err))
6775
return nil, err
@@ -76,7 +84,7 @@ func AdviseIndexes(ctx context.Context, sctx sessionctx.Context,
7684
advisorLogger().Info("indexable columns filled", zap.Int("indexable-cols", indexableColSet.Size()))
7785

7886
// start the advisor
79-
indexes, err := adviseIndexes(querySet, indexableColSet, option.MaxNumIndexes, option.MaxIndexWidth, opt)
87+
indexes, err := adviseIndexes(querySet, indexableColSet, opt, option)
8088
if err != nil {
8189
advisorLogger().Error("advise indexes failed", zap.Error(err))
8290
return nil, err
@@ -94,19 +102,19 @@ func AdviseIndexes(ctx context.Context, sctx sessionctx.Context,
94102

95103
// prepareQuerySet prepares the target queries for the index advisor.
96104
func prepareQuerySet(ctx context.Context, sctx sessionctx.Context,
97-
defaultDB string, opt Optimizer, specifiedSQLs []string) (s.Set[Query], error) {
105+
defaultDB string, opt Optimizer, option *Option) (s.Set[Query], error) {
98106
advisorLogger().Info("prepare target query set")
99107
querySet := s.NewSet[Query]()
100-
if len(specifiedSQLs) > 0 { // if target queries are specified
101-
for _, sql := range specifiedSQLs {
108+
if len(option.SpecifiedSQLs) > 0 { // if target queries are specified
109+
for _, sql := range option.SpecifiedSQLs {
102110
querySet.Add(Query{SchemaName: defaultDB, Text: sql, Frequency: 1})
103111
}
104112
} else {
105113
if intest.InTest && ctx.Value(TestKey("query_set")) != nil {
106114
querySet = ctx.Value(TestKey("query_set")).(s.Set[Query])
107115
} else {
108116
var err error
109-
if querySet, err = loadQuerySetFromStmtSummary(sctx); err != nil {
117+
if querySet, err = loadQuerySetFromStmtSummary(sctx, option); err != nil {
110118
return nil, err
111119
}
112120
if querySet.Size() == 0 {
@@ -117,24 +125,27 @@ func prepareQuerySet(ctx context.Context, sctx sessionctx.Context,
117125

118126
// filter invalid queries
119127
var err error
120-
querySet, err = RestoreSchemaName(defaultDB, querySet, len(specifiedSQLs) == 0)
128+
querySet, err = RestoreSchemaName(defaultDB, querySet, len(option.SpecifiedSQLs) == 0)
121129
if err != nil {
122130
return nil, err
123131
}
124-
querySet, err = FilterSQLAccessingSystemTables(querySet, len(specifiedSQLs) == 0)
132+
querySet, err = FilterSQLAccessingSystemTables(querySet, len(option.SpecifiedSQLs) == 0)
125133
if err != nil {
126134
return nil, err
127135
}
128-
querySet, err = FilterInvalidQueries(opt, querySet, len(specifiedSQLs) == 0)
136+
querySet, err = FilterInvalidQueries(opt, querySet, len(option.SpecifiedSQLs) == 0)
129137
if err != nil {
130138
return nil, err
131139
}
140+
if querySet.Size() == 0 {
141+
return nil, errors.New("empty query set after filtering invalid queries")
142+
}
132143
advisorLogger().Info("finish query preparation", zap.Int("num_query", querySet.Size()))
133144
return querySet, nil
134145
}
135146

136-
func loadQuerySetFromStmtSummary(sctx sessionctx.Context) (s.Set[Query], error) {
137-
sql := `SELECT any_value(schema_name) as schema_name,
147+
func loadQuerySetFromStmtSummary(sctx sessionctx.Context, option *Option) (s.Set[Query], error) {
148+
template := `SELECT any_value(schema_name) as schema_name,
138149
any_value(query_sample_text) as query_sample_text,
139150
sum(cast(exec_count as double)) as exec_count
140151
FROM information_schema.statements_summary_history
@@ -144,8 +155,8 @@ func loadQuerySetFromStmtSummary(sctx sessionctx.Context) (s.Set[Query], error)
144155
upper(schema_name) not in ("MYSQL", "INFORMATION_SCHEMA", "METRICS_SCHEMA", "PERFORMANCE_SCHEMA")
145156
GROUP BY digest
146157
ORDER BY sum(exec_count) DESC
147-
LIMIT 5000`
148-
rows, err := exec(sctx, sql)
158+
LIMIT %?`
159+
rows, err := exec(sctx, template, option.MaxNumQuery)
149160
if err != nil {
150161
return nil, err
151162
}

0 commit comments

Comments
 (0)