Skip to content

Commit 844038f

Browse files
qw4990ti-chi-bot
qw4990
authored and
ti-chi-bot
committed
This is an automated cherry-pick of pingcap#45962
Signed-off-by: ti-chi-bot <[email protected]>
1 parent 015241b commit 844038f

File tree

2 files changed

+299
-1
lines changed

2 files changed

+299
-1
lines changed

‎planner/core/planbuilder.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2884,7 +2884,7 @@ var CMSketchSizeLimit = kv.TxnEntrySizeLimit / binary.MaxVarintLen32
28842884

28852885
var analyzeOptionLimit = map[ast.AnalyzeOptionType]uint64{
28862886
ast.AnalyzeOptNumBuckets: 1024,
2887-
ast.AnalyzeOptNumTopN: 1024,
2887+
ast.AnalyzeOptNumTopN: 16384,
28882888
ast.AnalyzeOptCMSketchWidth: CMSketchSizeLimit,
28892889
ast.AnalyzeOptCMSketchDepth: CMSketchSizeLimit,
28902890
ast.AnalyzeOptNumSamples: 500000,

‎statistics/handle/handletest/analyze/analyze_test.go‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
// Copyright 2023 PingCAP, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package analyze
16+
17+
import (
18+
"bytes"
19+
"fmt"
20+
"testing"
21+
22+
"github.com/pingcap/failpoint"
23+
"github.com/pingcap/tidb/config"
24+
"github.com/pingcap/tidb/domain"
25+
"github.com/pingcap/tidb/parser/model"
26+
"github.com/pingcap/tidb/sessionctx/variable"
27+
"github.com/pingcap/tidb/statistics/handle"
28+
"github.com/pingcap/tidb/testkit"
29+
"github.com/stretchr/testify/require"
30+
)
31+
32+
// nolint:unused
33+
func checkForGlobalStatsWithOpts(t *testing.T, dom *domain.Domain, db, tt, pp string, topn, buckets int) {
34+
tbl, err := dom.InfoSchema().TableByName(model.NewCIStr(db), model.NewCIStr(tt))
35+
require.NoError(t, err)
36+
37+
tblInfo := tbl.Meta()
38+
physicalID := tblInfo.ID
39+
if pp != "global" {
40+
for _, def := range tbl.Meta().GetPartitionInfo().Definitions {
41+
if def.Name.L == pp {
42+
physicalID = def.ID
43+
}
44+
}
45+
}
46+
tblStats, err := dom.StatsHandle().TableStatsFromStorage(tblInfo, physicalID, true, 0)
47+
require.NoError(t, err)
48+
49+
delta := buckets/2 + 10
50+
for _, idxStats := range tblStats.Indices {
51+
if len(idxStats.Buckets) == 0 {
52+
continue // it's not loaded
53+
}
54+
numTopN := idxStats.TopN.Num()
55+
numBuckets := len(idxStats.Buckets)
56+
// since the hist-building algorithm doesn't stipulate the final bucket number to be equal to the expected number exactly,
57+
// we have to check the results by a range here.
58+
require.Equal(t, topn, numTopN)
59+
require.GreaterOrEqual(t, numBuckets, buckets-delta)
60+
require.LessOrEqual(t, numBuckets, buckets+delta)
61+
}
62+
for _, colStats := range tblStats.Columns {
63+
if len(colStats.Buckets) == 0 {
64+
continue // it's not loaded
65+
}
66+
numTopN := colStats.TopN.Num()
67+
numBuckets := len(colStats.Buckets)
68+
require.Equal(t, topn, numTopN)
69+
require.GreaterOrEqual(t, numBuckets, buckets-delta)
70+
require.LessOrEqual(t, numBuckets, buckets+delta)
71+
}
72+
}
73+
74+
// nolint:unused
75+
func prepareForGlobalStatsWithOptsV2(t *testing.T, dom *domain.Domain, tk *testkit.TestKit, tblName, dbName string) {
76+
tk.MustExec("create database if not exists " + dbName)
77+
tk.MustExec("use " + dbName)
78+
tk.MustExec("drop table if exists " + tblName)
79+
tk.MustExec(` create table ` + tblName + ` (a int, key(a)) partition by range (a) ` +
80+
`(partition p0 values less than (100000), partition p1 values less than (200000))`)
81+
buf1 := bytes.NewBufferString("insert into " + tblName + " values (0)")
82+
buf2 := bytes.NewBufferString("insert into " + tblName + " values (100000)")
83+
for i := 0; i < 1000; i++ {
84+
buf1.WriteString(fmt.Sprintf(", (%v)", 2))
85+
buf2.WriteString(fmt.Sprintf(", (%v)", 100002))
86+
buf1.WriteString(fmt.Sprintf(", (%v)", 1))
87+
buf2.WriteString(fmt.Sprintf(", (%v)", 100001))
88+
buf1.WriteString(fmt.Sprintf(", (%v)", 0))
89+
buf2.WriteString(fmt.Sprintf(", (%v)", 100000))
90+
}
91+
for i := 0; i < 5000; i += 3 {
92+
buf1.WriteString(fmt.Sprintf(", (%v)", i))
93+
buf2.WriteString(fmt.Sprintf(", (%v)", 100000+i))
94+
}
95+
tk.MustExec(buf1.String())
96+
tk.MustExec(buf2.String())
97+
tk.MustExec("set @@tidb_analyze_version=2")
98+
tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
99+
require.NoError(t, dom.StatsHandle().DumpStatsDeltaToKV(handle.DumpAll))
100+
}
101+
102+
// nolint:unused
103+
func prepareForGlobalStatsWithOpts(t *testing.T, dom *domain.Domain, tk *testkit.TestKit, tblName, dbName string) {
104+
tk.MustExec("create database if not exists " + dbName)
105+
tk.MustExec("use " + dbName)
106+
tk.MustExec("drop table if exists " + tblName)
107+
tk.MustExec(` create table ` + tblName + ` (a int, key(a)) partition by range (a) ` +
108+
`(partition p0 values less than (100000), partition p1 values less than (200000))`)
109+
buf1 := bytes.NewBufferString("insert into " + tblName + " values (0)")
110+
buf2 := bytes.NewBufferString("insert into " + tblName + " values (100000)")
111+
for i := 0; i < 5000; i += 3 {
112+
buf1.WriteString(fmt.Sprintf(", (%v)", i))
113+
buf2.WriteString(fmt.Sprintf(", (%v)", 100000+i))
114+
}
115+
for i := 0; i < 1000; i++ {
116+
buf1.WriteString(fmt.Sprintf(", (%v)", 0))
117+
buf2.WriteString(fmt.Sprintf(", (%v)", 100000))
118+
}
119+
tk.MustExec(buf1.String())
120+
tk.MustExec(buf2.String())
121+
tk.MustExec("set @@tidb_analyze_version=2")
122+
tk.MustExec("set @@tidb_partition_prune_mode='dynamic'")
123+
require.NoError(t, dom.StatsHandle().DumpStatsDeltaToKV(handle.DumpAll))
124+
}
125+
126+
func TestAnalyzeVirtualCol(t *testing.T) {
127+
store := testkit.CreateMockStore(t)
128+
tk := testkit.NewTestKit(t, store)
129+
tk.MustExec("use test")
130+
tk.MustExec("drop table if exists t")
131+
tk.MustExec("create table t(a int, b int generated always as (-a) virtual, c int generated always as (-a) stored, index (c))")
132+
tk.MustExec("insert into t(a) values(2),(1),(1),(3),(NULL)")
133+
tk.MustExec("set @@tidb_analyze_version = 2")
134+
tk.MustExec("analyze table t")
135+
require.Len(t, tk.MustQuery("show stats_histograms where table_name ='t'").Rows(), 3)
136+
}
137+
138+
func TestAnalyzeGlobalStatsWithOpts1(t *testing.T) {
139+
store, dom := testkit.CreateMockStoreAndDomain(t)
140+
tk := testkit.NewTestKit(t, store)
141+
prepareForGlobalStatsWithOpts(t, dom, tk, "test_gstats_opt", "test_gstats_opt")
142+
143+
// nolint:unused
144+
type opt struct {
145+
topn int
146+
buckets int
147+
err bool
148+
}
149+
150+
cases := []opt{
151+
{1, 37, false},
152+
{2, 47, false},
153+
{10, 77, false},
154+
{77, 219, false},
155+
{-31, 222, true},
156+
{10, -77, true},
157+
{100000, 47, true},
158+
{77, 47000, true},
159+
}
160+
for _, ca := range cases {
161+
sql := fmt.Sprintf("analyze table test_gstats_opt with %v topn, %v buckets", ca.topn, ca.buckets)
162+
if !ca.err {
163+
tk.MustExec(sql)
164+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt", "test_gstats_opt", "global", ca.topn, ca.buckets)
165+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt", "test_gstats_opt", "p0", ca.topn, ca.buckets)
166+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt", "test_gstats_opt", "p1", ca.topn, ca.buckets)
167+
} else {
168+
err := tk.ExecToErr(sql)
169+
require.Error(t, err)
170+
}
171+
}
172+
}
173+
174+
func TestAnalyzeGlobalStatsWithOpts2(t *testing.T) {
175+
store, dom := testkit.CreateMockStoreAndDomain(t)
176+
tk := testkit.NewTestKit(t, store)
177+
originalVal1 := tk.MustQuery("select @@tidb_persist_analyze_options").Rows()[0][0].(string)
178+
defer func() {
179+
tk.MustExec(fmt.Sprintf("set global tidb_persist_analyze_options = %v", originalVal1))
180+
}()
181+
tk.MustExec("set global tidb_persist_analyze_options=false")
182+
prepareForGlobalStatsWithOptsV2(t, dom, tk, "test_gstats_opt2", "test_gstats_opt2")
183+
184+
tk.MustExec("analyze table test_gstats_opt2 with 2 topn, 10 buckets, 1000 samples")
185+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "global", 2, 10)
186+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p0", 2, 10)
187+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p1", 2, 10)
188+
189+
// analyze a partition to let its options be different with others'
190+
tk.MustExec("analyze table test_gstats_opt2 partition p0 with 3 topn, 20 buckets")
191+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "global", 3, 20) // use new options
192+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p0", 3, 20)
193+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p1", 2, 10)
194+
195+
tk.MustExec("analyze table test_gstats_opt2 partition p1 with 1 topn, 15 buckets")
196+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "global", 1, 15)
197+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p0", 3, 20)
198+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p1", 1, 15)
199+
200+
tk.MustExec("analyze table test_gstats_opt2 partition p0 with 2 topn, 10 buckets") // change back to 2 topn
201+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "global", 2, 10)
202+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p0", 2, 10)
203+
checkForGlobalStatsWithOpts(t, dom, "test_gstats_opt2", "test_gstats_opt2", "p1", 1, 15)
204+
}
205+
206+
func TestAnalyzeWithDynamicPartitionPruneMode(t *testing.T) {
207+
store := testkit.CreateMockStore(t)
208+
tk := testkit.NewTestKit(t, store)
209+
tk.MustExec("use test")
210+
tk.MustExec("set @@tidb_partition_prune_mode = '" + string(variable.Dynamic) + "'")
211+
tk.MustExec("set @@tidb_analyze_version = 2")
212+
tk.MustExec(`create table t (a int, key(a)) partition by range(a)
213+
(partition p0 values less than (10),
214+
partition p1 values less than (22))`)
215+
tk.MustExec(`insert into t values (1), (2), (3), (10), (11)`)
216+
tk.MustExec(`analyze table t with 1 topn, 2 buckets`)
217+
rows := tk.MustQuery("show stats_buckets where partition_name = 'global' and is_index=1").Rows()
218+
require.Len(t, rows, 2)
219+
require.Equal(t, "4", rows[1][6])
220+
tk.MustExec("insert into t values (1), (2), (2)")
221+
tk.MustExec("analyze table t partition p0 with 1 topn, 2 buckets")
222+
rows = tk.MustQuery("show stats_buckets where partition_name = 'global' and is_index=1").Rows()
223+
require.Len(t, rows, 2)
224+
require.Equal(t, "5", rows[1][6])
225+
tk.MustExec("insert into t values (3)")
226+
tk.MustExec("analyze table t partition p0 index a with 1 topn, 2 buckets")
227+
rows = tk.MustQuery("show stats_buckets where partition_name = 'global' and is_index=1").Rows()
228+
require.Len(t, rows, 1)
229+
require.Equal(t, "6", rows[0][6])
230+
}
231+
232+
func TestFMSWithAnalyzePartition(t *testing.T) {
233+
store := testkit.CreateMockStore(t)
234+
tk := testkit.NewTestKit(t, store)
235+
tk.MustExec("use test")
236+
tk.MustExec("set @@tidb_partition_prune_mode = '" + string(variable.Dynamic) + "'")
237+
tk.MustExec("set @@tidb_analyze_version = 2")
238+
tk.MustExec(`create table t (a int, key(a)) partition by range(a)
239+
(partition p0 values less than (10),
240+
partition p1 values less than (22))`)
241+
tk.MustExec(`insert into t values (1), (2), (3), (10), (11)`)
242+
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("0"))
243+
tk.MustExec("analyze table t partition p0 with 1 topn, 2 buckets")
244+
tk.MustQuery("show warnings").Sort().Check(testkit.Rows(
245+
"Note 1105 Analyze use auto adjusted sample rate 1.000000 for table test.t's partition p0, reason to use this rate is \"use min(1, 110000/10000) as the sample-rate=1\"",
246+
"Warning 1105 Ignore columns and options when analyze partition in dynamic mode",
247+
"Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
248+
"Warning 8131 Build global-level stats failed due to missing partition-level stats: table `t` partition `p1`",
249+
))
250+
tk.MustQuery("select count(*) from mysql.stats_fm_sketch").Check(testkit.Rows("2"))
251+
}
252+
253+
func TestFastAnalyzeColumnHistWithNullValue(t *testing.T) {
254+
store := testkit.CreateMockStore(t)
255+
testKit := testkit.NewTestKit(t, store)
256+
testKit.MustExec("use test")
257+
testKit.MustExec("drop table if exists t")
258+
testKit.MustExec("create table t (a int)")
259+
testKit.MustExec("insert into t values (1), (2), (3), (4), (NULL)")
260+
testKit.MustExec("set @@session.tidb_analyze_version = 1")
261+
testKit.MustExec("set @@tidb_enable_fast_analyze=1")
262+
defer testKit.MustExec("set @@tidb_enable_fast_analyze=0")
263+
testKit.MustExec("analyze table t with 0 topn, 2 buckets")
264+
// If NULL is in hist, the min(lower_bound) will be "".
265+
testKit.MustQuery("select min(lower_bound) from mysql.stats_buckets").Check(testkit.Rows("1"))
266+
}
267+
268+
func TestAnalyzeIncrementalEvictedIndex(t *testing.T) {
269+
t.Skip("now we don't support to evict index")
270+
restore := config.RestoreFunc()
271+
defer restore()
272+
config.UpdateGlobal(func(conf *config.Config) {
273+
conf.Performance.EnableStatsCacheMemQuota = true
274+
})
275+
store, dom := testkit.CreateMockStoreAndDomain(t)
276+
tk := testkit.NewTestKit(t, store)
277+
tk.MustExec("set @@tidb_analyze_version = 1")
278+
tk.MustExec("use test")
279+
tk.MustExec("drop table if exists t")
280+
tk.MustExec("create table t(a int, b varchar(10), index idx_b (b))")
281+
tk.MustExec("analyze table test.t")
282+
tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
283+
require.Nil(t, err)
284+
tblStats := domain.GetDomain(tk.Session()).StatsHandle().GetTableStats(tbl.Meta())
285+
for _, index := range tblStats.Indices {
286+
require.False(t, index.IsEvicted())
287+
}
288+
289+
domain.GetDomain(tk.Session()).StatsHandle().SetStatsCacheCapacity(1)
290+
tblStats = domain.GetDomain(tk.Session()).StatsHandle().GetTableStats(tbl.Meta())
291+
for _, index := range tblStats.Indices {
292+
require.True(t, index.IsEvicted())
293+
}
294+
295+
require.Nil(t, failpoint.Enable("github.com/pingcap/tidb/executor/assertEvictIndex", `return(true)`))
296+
tk.MustExec("analyze incremental table test.t index idx_b")
297+
require.Nil(t, failpoint.Disable("github.com/pingcap/tidb/executor/assertEvictIndex"))
298+
}

0 commit comments

Comments
 (0)