planner: constant folding to isnull(not null column) (pingcap#62163)

ref pingcap#7973, close pingcap#62050

Author: ti-chi-bot

pkg/planner/core/casetest/rule/BUILD.bazel

@@ -10,10 +10,11 @@ go_test(
         "rule_join_reorder_test.go",
         "rule_outer2inner_test.go",
         "rule_predicate_pushdown_test.go",
+        "rule_predicate_simplification_test.go",
     ],
     data = glob(["testdata/**"]),
     flaky = True,
-    shard_count = 8,
+    shard_count = 9,
     deps = [
         "//pkg/domain",
         "//pkg/expression",

pkg/planner/core/casetest/rule/main_test.go

@@ -33,6 +33,7 @@ func TestMain(m *testing.M) {
 	testDataMap.LoadTestSuiteData("testdata", "derive_topn_from_window")
 	testDataMap.LoadTestSuiteData("testdata", "join_reorder_suite")
 	testDataMap.LoadTestSuiteData("testdata", "predicate_pushdown_suite")
+	testDataMap.LoadTestSuiteData("testdata", "predicate_simplification")
 	opts := []goleak.Option{
 		goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
 		goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"),
@@ -66,3 +67,7 @@ func GetJoinReorderSuiteData() testdata.TestData {
 func GetPredicatePushdownSuiteData() testdata.TestData {
 	return testDataMap["predicate_pushdown_suite"]
 }
+
+func GetPredicateSimplificationSuiteData() testdata.TestData {
+	return testDataMap["predicate_simplification"]
+}

pkg/planner/core/casetest/rule/rule_predicate_simplification_test.go

@@ -0,0 +1,74 @@
+// Copyright 2025 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rule
+
+import (
+	"testing"
+
+	"github.com/pingcap/tidb/pkg/testkit"
+	"github.com/pingcap/tidb/pkg/testkit/testdata"
+)
+
+func TestPredicateSimplification(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec(`CREATE TABLE t1 (
+    id VARCHAR(64) PRIMARY KEY
+);`)
+	tk.MustExec(`CREATE TABLE t2 (
+    c1 VARCHAR(64) NOT NULL,
+    c2 VARCHAR(64) NOT NULL,
+    c3 VARCHAR(64) NOT NULL,
+    PRIMARY KEY (c1, c2, c3),
+    KEY c3 (c3)
+);`)
+	tk.MustExec(`CREATE TABLE t3 (
+    c1 VARCHAR(64) NOT NULL,
+    c2 VARCHAR(64) NOT NULL,
+    c3 VARCHAR(64) NOT NULL,
+    PRIMARY KEY (c1, c2, c3),
+    KEY c3 (c3)
+);`)
+	tk.MustExec(`CREATE TABLE t4 (
+    c1 VARCHAR(64) NOT NULL,
+    c2 VARCHAR(64) NOT NULL,
+    c3 VARCHAR(64) NOT NULL,
+    state VARCHAR(64) NOT NULL DEFAULT 'ACTIVE',
+    PRIMARY KEY (c1, c2, c3),
+    KEY c3 (c3)
+);`)
+	tk.MustExec(`CREATE TABLE t5 (
+    c1 VARCHAR(64) NOT NULL,
+    c2 VARCHAR(64) NOT NULL,
+    PRIMARY KEY (c1, c2)
+);`)
+	// since the plan may differ under different planner mode, recommend to record explain result to json accordingly.
+	var input []string
+	var output []struct {
+		SQL  string
+		Plan []string
+	}
+	suite := GetPredicateSimplificationSuiteData()
+	suite.LoadTestCases(t, &input, &output)
+	for i, tt := range input {
+		testdata.OnRecord(func() {
+			output[i].SQL = tt
+			output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery("explain format=brief " + tt).Rows())
+		})
+		res := tk.MustQuery("explain format=brief " + tt)
+		res.Check(testkit.Rows(output[i].Plan...))
+	}
+}

pkg/planner/core/casetest/rule/testdata/predicate_simplification_in.json

@@ -0,0 +1,8 @@
+[
+  {
+    "name": "TestPredicateSimplification",
+    "cases": [
+      "SELECT i.id, ip_products.products FROM t1 AS i LEFT JOIN t4 ON i.id = t4.c3 LEFT JOIN (SELECT t4.c3, GROUP_CONCAT(DISTINCT t2.c3 ORDER BY t2.c3 ASC) AS products FROM t4 JOIN t3 ON t4.c1 = t3.c1 AND t4.c2 = t3.c2 LEFT JOIN t2 ON t4.c1 = t2.c1 AND t4.c2 = t2.c2 WHERE t3.c3 = 'production' AND t4.state = 'ACTIVE' GROUP BY t4.c3, t4.c1, t4.c2) AS ip_products ON t4.c3 = ip_products.c3 LEFT JOIN t5 ON i.id = t5.c1 AND t5.c2 = 'production' WHERE t4.state = 'ACTIVE' AND t5.c1 IS NULL GROUP BY i.id, ip_products.products HAVING FIND_IN_SET('info', products) ORDER BY i.id ASC LIMIT 500 OFFSET 5500;"
+    ]
+  }
+]

pkg/planner/core/casetest/rule/testdata/predicate_simplification_out.json

@@ -0,0 +1,38 @@
+[
+  {
+    "Name": "TestPredicateSimplification",
+    "Cases": [
+      {
+        "SQL": "SELECT i.id, ip_products.products FROM t1 AS i LEFT JOIN t4 ON i.id = t4.c3 LEFT JOIN (SELECT t4.c3, GROUP_CONCAT(DISTINCT t2.c3 ORDER BY t2.c3 ASC) AS products FROM t4 JOIN t3 ON t4.c1 = t3.c1 AND t4.c2 = t3.c2 LEFT JOIN t2 ON t4.c1 = t2.c1 AND t4.c2 = t2.c2 WHERE t3.c3 = 'production' AND t4.state = 'ACTIVE' GROUP BY t4.c3, t4.c1, t4.c2) AS ip_products ON t4.c3 = ip_products.c3 LEFT JOIN t5 ON i.id = t5.c1 AND t5.c2 = 'production' WHERE t4.state = 'ACTIVE' AND t5.c1 IS NULL GROUP BY i.id, ip_products.products HAVING FIND_IN_SET('info', products) ORDER BY i.id ASC LIMIT 500 OFFSET 5500;",
+        "Plan": [
+          "TopN 8.00 root  test.t1.id, offset:5500, count:500",
+          "└─HashAgg 8.00 root  group by:Column#16, test.t1.id, funcs:firstrow(test.t1.id)->test.t1.id, funcs:firstrow(Column#16)->Column#16",
+          "  └─Selection 8.00 root  isnull(test.t5.c1)",
+          "    └─Projection 10.00 root  test.t1.id, Column#16, test.t5.c1",
+          "      └─HashJoin 10.00 root  inner join, equal:[eq(test.t4.c3, test.t4.c3)]",
+          "        ├─Selection(Build) 6.40 root  find_in_set(\"info\", Column#16)",
+          "        │ └─HashAgg 8.00 root  group by:test.t4.c1, test.t4.c2, test.t4.c3, funcs:group_concat(distinct test.t2.c3 order by test.t2.c3 separator \",\")->Column#16, funcs:firstrow(test.t4.c3)->test.t4.c3",
+          "        │   └─IndexJoin 15.62 root  left outer join, inner:TableReader, outer key:test.t4.c1, test.t4.c2, inner key:test.t2.c1, test.t2.c2, equal cond:eq(test.t4.c1, test.t2.c1), eq(test.t4.c2, test.t2.c2)",
+          "        │     ├─IndexJoin(Build) 12.50 root  inner join, inner:TableReader, outer key:test.t3.c1, test.t3.c2, inner key:test.t4.c1, test.t4.c2, equal cond:eq(test.t3.c1, test.t4.c1), eq(test.t3.c2, test.t4.c2)",
+          "        │     │ ├─IndexReader(Build) 10.00 root  index:IndexRangeScan",
+          "        │     │ │ └─IndexRangeScan 10.00 cop[tikv] table:t3, index:c3(c3) range:[\"production\",\"production\"], keep order:false, stats:pseudo",
+          "        │     │ └─TableReader(Probe) 0.01 root  data:Selection",
+          "        │     │   └─Selection 0.01 cop[tikv]  eq(test.t4.state, \"ACTIVE\")",
+          "        │     │     └─TableRangeScan 10.00 cop[tikv] table:t4 range: decided by [eq(test.t4.c1, test.t3.c1) eq(test.t4.c2, test.t3.c2)], keep order:false, stats:pseudo",
+          "        │     └─TableReader(Probe) 12.50 root  data:TableRangeScan",
+          "        │       └─TableRangeScan 12.50 cop[tikv] table:t2 range: decided by [eq(test.t2.c1, test.t4.c1) eq(test.t2.c2, test.t4.c2)], keep order:false, stats:pseudo",
+          "        └─IndexJoin(Probe) 12.50 root  left outer join, inner:TableReader, outer key:test.t1.id, inner key:test.t5.c1, equal cond:eq(test.t1.id, test.t5.c1)",
+          "          ├─IndexJoin(Build) 12.50 root  inner join, inner:TableReader, outer key:test.t4.c3, inner key:test.t1.id, equal cond:eq(test.t4.c3, test.t1.id)",
+          "          │ ├─TableReader(Build) 10.00 root  data:Selection",
+          "          │ │ └─Selection 10.00 cop[tikv]  eq(test.t4.state, \"ACTIVE\")",
+          "          │ │   └─TableFullScan 10000.00 cop[tikv] table:t4 keep order:false, stats:pseudo",
+          "          │ └─TableReader(Probe) 10.00 root  data:TableRangeScan",
+          "          │   └─TableRangeScan 10.00 cop[tikv] table:i range: decided by [eq(test.t1.id, test.t4.c3)], keep order:false, stats:pseudo",
+          "          └─TableReader(Probe) 0.01 root  data:Selection",
+          "            └─Selection 0.01 cop[tikv]  eq(test.t5.c2, \"production\")",
+          "              └─TableRangeScan 12.50 cop[tikv] table:t5 range: decided by [eq(test.t5.c1, test.t1.id) eq(test.t5.c2, production)], keep order:false, stats:pseudo"
+        ]
+      }
+    ]
+  }
+]

pkg/planner/core/constraint/BUILD.bazel

@@ -7,6 +7,8 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/expression",
+        "//pkg/parser/ast",
+        "//pkg/parser/mysql",
         "//pkg/planner/core/base",
     ],
 )

pkg/planner/core/constraint/exprs.go

@@ -15,7 +15,11 @@
 package constraint
 
 import (
+	"slices"
+
 	"github.com/pingcap/tidb/pkg/expression"
+	"github.com/pingcap/tidb/pkg/parser/ast"
+	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/planner/core/base"
 )
 
@@ -40,3 +44,31 @@ func DeleteTrueExprs(p base.LogicalPlan, conds []expression.Expression) []expres
 	}
 	return newConds
 }
+
+// DeleteTrueExprsBySchema delete true expressions such as not(isnull(not null column)).
+// It is used in the predicate pushdown optimization to remove unnecessary conditions which will be pushed down to child operators.
+func DeleteTrueExprsBySchema(ctx expression.EvalContext, schema *expression.Schema, conds []expression.Expression) []expression.Expression {
+	return slices.DeleteFunc(conds, func(item expression.Expression) bool {
+		if expr, ok := item.(*expression.ScalarFunction); ok && expr.FuncName.L == ast.UnaryNot {
+			if args := expr.GetArgs(); len(args) == 1 {
+				// If the expression is `not(isnull(not null column))`, we can remove it.
+				return isNullWithNotNullColumn(ctx, schema, args[0])
+			}
+		}
+		return false
+	})
+}
+
+// isNullWithNotNullColumn checks if the expression is `isnull(not null column)`.
+func isNullWithNotNullColumn(ctx expression.EvalContext, schema *expression.Schema, expr expression.Expression) bool {
+	if e, ok := expr.(*expression.ScalarFunction); ok && e.FuncName.L == ast.IsNull {
+		if args := e.GetArgs(); len(args) == 1 {
+			if col, ok := args[0].(*expression.Column); ok {
+				if retrieveColumn := schema.RetrieveColumn(col); retrieveColumn != nil {
+					return mysql.HasNotNullFlag(retrieveColumn.GetType(ctx).GetFlag())
+				}
+			}
+		}
+	}
+	return false
+}

pkg/planner/core/operator/logicalop/logical_join.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/planner/cardinality"
 	"github.com/pingcap/tidb/pkg/planner/core/base"
+	"github.com/pingcap/tidb/pkg/planner/core/constraint"
 	"github.com/pingcap/tidb/pkg/planner/core/cost"
 	ruleutil "github.com/pingcap/tidb/pkg/planner/core/rule/util"
 	"github.com/pingcap/tidb/pkg/planner/funcdep"
@@ -282,8 +283,14 @@ func (p *LogicalJoin) PredicatePushDown(predicates []expression.Expression, opt
 	}
 	leftCond = expression.RemoveDupExprs(leftCond)
 	rightCond = expression.RemoveDupExprs(rightCond)
-	leftRet, lCh := p.Children()[0].PredicatePushDown(leftCond, opt)
-	rightRet, rCh := p.Children()[1].PredicatePushDown(rightCond, opt)
+	evalCtx := p.SCtx().GetExprCtx().GetEvalCtx()
+	children := p.Children()
+	rightChild := children[1]
+	leftChild := children[0]
+	rightCond = constraint.DeleteTrueExprsBySchema(evalCtx, rightChild.Schema(), rightCond)
+	leftCond = constraint.DeleteTrueExprsBySchema(evalCtx, leftChild.Schema(), leftCond)
+	leftRet, lCh := leftChild.PredicatePushDown(leftCond, opt)
+	rightRet, rCh := rightChild.PredicatePushDown(rightCond, opt)
 	addSelection(p, lCh, leftRet, 0, opt)
 	addSelection(p, rCh, rightRet, 1, opt)
 	p.updateEQCond()