planner: fix the wrong join estimation depending on missing or uninitialized stats (#61604) (#61676)

close #61602

Author: ti-chi-bot

pkg/planner/core/casetest/physicalplantest/testdata/plan_suite_out.json

@@ -4264,10 +4264,10 @@
       {
         "SQL": "SELECT col2 FROM PK_S_MULTI_31 AS T1 WHERE (SELECT count(DISTINCT COL1, COL2) FROM PK_S_MULTI_31 AS T2 WHERE T2.COL1>T1.COL1)>2 order by col2;",
         "Plan": [
-          "Sort 0.80 root  test.pk_s_multi_31.col2",
-          "└─Projection 0.80 root  test.pk_s_multi_31.col2",
-          "  └─Selection 0.80 root  gt(Column#7, 2)",
-          "    └─HashAgg 1.00 root  group by:test.pk_s_multi_31.col1, test.pk_s_multi_31.col2, funcs:firstrow(test.pk_s_multi_31.col2)->test.pk_s_multi_31.col2, funcs:count(distinct test.pk_s_multi_31.col1, test.pk_s_multi_31.col2)->Column#7",
+          "Sort 6400.00 root  test.pk_s_multi_31.col2",
+          "└─Projection 6400.00 root  test.pk_s_multi_31.col2",
+          "  └─Selection 6400.00 root  gt(Column#7, 2)",
+          "    └─HashAgg 8000.00 root  group by:test.pk_s_multi_31.col1, test.pk_s_multi_31.col2, funcs:firstrow(test.pk_s_multi_31.col2)->test.pk_s_multi_31.col2, funcs:count(distinct test.pk_s_multi_31.col1, test.pk_s_multi_31.col2)->Column#7",
           "      └─HashJoin 100000000.00 root  CARTESIAN left outer join, other cond:gt(test.pk_s_multi_31.col1, test.pk_s_multi_31.col1)",
           "        ├─IndexReader(Build) 10000.00 root  index:IndexFullScan",
           "        │ └─IndexFullScan 10000.00 cop[tikv] table:T2, index:PRIMARY(COL1, COL2) keep order:false, stats:pseudo",

pkg/planner/core/stats.go

@@ -200,7 +200,7 @@ func (ds *DataSource) getGroupNDVs(colGroups [][]*expression.Column) []property.
 					break
 				}
 			}
-			if match {
+			if match && idx.IsEssentialStatsLoaded() {
 				ndv := property.GroupNDV{
 					Cols: idxCols,
 					NDV:  float64(idx.NDV),

tests/integrationtest/r/explain_complex.result

@@ -153,16 +153,16 @@ Projection	1.00	root		explain_complex.st.cm, explain_complex.st.p1, explain_comp
       └─TableRowIDScan	250.00	cop[tikv]	table:st	keep order:false, stats:pseudo
 explain format = 'brief' select dt.id as id, dt.aid as aid, dt.pt as pt, dt.dic as dic, dt.cm as cm, rr.gid as gid, rr.acd as acd, rr.t as t,dt.p1 as p1, dt.p2 as p2, dt.p3 as p3, dt.p4 as p4, dt.p5 as p5, dt.p6_md5 as p6, dt.p7_md5 as p7 from dt dt join rr rr on (rr.pt = 'ios' and rr.t > 1478185592 and dt.aid = rr.aid and dt.dic = rr.dic) where dt.pt = 'ios' and dt.t > 1478185592 and dt.bm = 0 limit 2000;
 id	estRows	task	access object	operator info
-Projection	0.01	root		explain_complex.dt.id, explain_complex.dt.aid, explain_complex.dt.pt, explain_complex.dt.dic, explain_complex.dt.cm, explain_complex.rr.gid, explain_complex.rr.acd, explain_complex.rr.t, explain_complex.dt.p1, explain_complex.dt.p2, explain_complex.dt.p3, explain_complex.dt.p4, explain_complex.dt.p5, explain_complex.dt.p6_md5, explain_complex.dt.p7_md5
-└─Limit	0.01	root		offset:0, count:2000
-  └─IndexJoin	0.01	root		inner join, inner:IndexLookUp, outer key:explain_complex.rr.aid, explain_complex.rr.dic, inner key:explain_complex.dt.aid, explain_complex.dt.dic, equal cond:eq(explain_complex.rr.aid, explain_complex.dt.aid), eq(explain_complex.rr.dic, explain_complex.dt.dic)
+Projection	0.00	root		explain_complex.dt.id, explain_complex.dt.aid, explain_complex.dt.pt, explain_complex.dt.dic, explain_complex.dt.cm, explain_complex.rr.gid, explain_complex.rr.acd, explain_complex.rr.t, explain_complex.dt.p1, explain_complex.dt.p2, explain_complex.dt.p3, explain_complex.dt.p4, explain_complex.dt.p5, explain_complex.dt.p6_md5, explain_complex.dt.p7_md5
+└─Limit	0.00	root		offset:0, count:2000
+  └─IndexJoin	0.00	root		inner join, inner:IndexLookUp, outer key:explain_complex.rr.aid, explain_complex.rr.dic, inner key:explain_complex.dt.aid, explain_complex.dt.dic, equal cond:eq(explain_complex.rr.aid, explain_complex.dt.aid), eq(explain_complex.rr.dic, explain_complex.dt.dic)
     ├─TableReader(Build)	3.33	root		data:Selection
     │ └─Selection	3.33	cop[tikv]		eq(explain_complex.rr.pt, "ios"), gt(explain_complex.rr.t, 1478185592)
     │   └─TableFullScan	10000.00	cop[tikv]	table:rr	keep order:false, stats:pseudo
-    └─IndexLookUp(Probe)	0.01	root		
+    └─IndexLookUp(Probe)	0.00	root		
       ├─Selection(Build)	3.33	cop[tikv]		not(isnull(explain_complex.dt.dic))
       │ └─IndexRangeScan	3.33	cop[tikv]	table:dt, index:aid(aid, dic)	range: decided by [eq(explain_complex.dt.aid, explain_complex.rr.aid) eq(explain_complex.dt.dic, explain_complex.rr.dic)], keep order:false, stats:pseudo
-      └─Selection(Probe)	0.01	cop[tikv]		eq(explain_complex.dt.bm, 0), eq(explain_complex.dt.pt, "ios"), gt(explain_complex.dt.t, 1478185592)
+      └─Selection(Probe)	0.00	cop[tikv]		eq(explain_complex.dt.bm, 0), eq(explain_complex.dt.pt, "ios"), gt(explain_complex.dt.t, 1478185592)
         └─TableRowIDScan	3.33	cop[tikv]	table:dt	keep order:false, stats:pseudo
 explain format = 'brief' select pc,cr,count(DISTINCT uid) as pay_users,count(oid) as pay_times,sum(am) as am from pp where ps=2  and ppt>=1478188800 and ppt<1478275200  and pi in ('510017','520017') and uid in ('18089709','18090780') group by pc,cr;
 id	estRows	task	access object	operator info