statsitstics: avoid sync load column which is skiped type to analyze (#57144)

pkg/statistics/handle/handle_hist.go

@@ -28,6 +28,7 @@ import (
 	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
+	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/statistics/handle/storage"
 	utilstats "github.com/pingcap/tidb/pkg/statistics/handle/util"
@@ -280,6 +281,19 @@ func (h *Handle) handleOneItemTask(task *NeededItemTask) (err error) {
 			h.SPool().Put(se)
 		}
 	}()
+<<<<<<< HEAD:pkg/statistics/handle/handle_hist.go
+=======
+	var skipTypes map[string]struct{}
+	val, err := sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(variable.TiDBAnalyzeSkipColumnTypes)
+	if err != nil {
+		logutil.BgLogger().Warn("failed to get global variable", zap.Error(err))
+	} else {
+		skipTypes = variable.ParseAnalyzeSkipColumnTypes(val)
+	}
+
+	item := task.Item.TableItemID
+	tbl, ok := s.statsHandle.Get(item.TableID)
+>>>>>>> bfec7325a12 (statsitstics: avoid sync load column which is skiped type to analyze (#57144)):pkg/statistics/handle/syncload/stats_syncload.go
 
 	item := task.TableItemID
 	tbl, ok := h.Get(item.TableID)
@@ -294,12 +308,44 @@ func (h *Handle) handleOneItemTask(task *NeededItemTask) (err error) {
 		}
 		wrapper.idx = index
 	} else {
+<<<<<<< HEAD:pkg/statistics/handle/handle_hist.go
 		col, ok := tbl.Columns[item.ID]
 		if !ok || col.IsFullLoad() {
+=======
+		col, loadNeeded, analyzed := tbl.ColumnIsLoadNeeded(item.ID, task.Item.FullLoad)
+		if !loadNeeded {
+			return nil
+		}
+		if col != nil {
+			wrapper.colInfo = col.Info
+		} else {
+			// Now, we cannot init the column info in the ColAndIdxExistenceMap when to disable lite-init-stats.
+			// so we have to get the column info from the domain.
+			wrapper.colInfo = tblInfo.Meta().GetColumnByID(item.ID)
+		}
+		if skipTypes != nil {
+			_, skip := skipTypes[types.TypeToStr(wrapper.colInfo.FieldType.GetType(), wrapper.colInfo.FieldType.GetCharset())]
+			if skip {
+				return nil
+			}
+		}
+
+		// If this column is not analyzed yet and we don't have it in memory.
+		// We create a fake one for the pseudo estimation.
+		if loadNeeded && !analyzed {
+			wrapper.col = &statistics.Column{
+				PhysicalID: item.TableID,
+				Info:       wrapper.colInfo,
+				Histogram:  *statistics.NewHistogram(item.ID, 0, 0, 0, &wrapper.colInfo.FieldType, 0, 0),
+				IsHandle:   isPkIsHandle && mysql.HasPriKeyFlag(wrapper.colInfo.GetFlag()),
+			}
+			s.updateCachedItem(tblInfo, item, wrapper.col, wrapper.idx, task.Item.FullLoad)
+>>>>>>> bfec7325a12 (statsitstics: avoid sync load column which is skiped type to analyze (#57144)):pkg/statistics/handle/syncload/stats_syncload.go
 			return nil
 		}
 		wrapper.col = col
 	}
+	failpoint.Inject("handleOneItemTaskPanic", nil)
 	t := time.Now()
 	needUpdate := false
 	wrapper, err = h.readStatsForOneItem(sctx, item, wrapper)

pkg/statistics/handle/handle_hist_test.go

@@ -51,6 +51,22 @@ func TestSyncLoadSkipUnAnalyzedItems(t *testing.T) {
 	failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/assertSyncLoadItems")
 }
 
+func TestSyncLoadSkipAnalyzSkipColumnItems(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(`id` bigint(20) NOT NULL AUTO_INCREMENT,content text,PRIMARY KEY (`id`))")
+	h := dom.StatsHandle()
+	h.SetLease(1)
+
+	tk.MustExec("analyze table t")
+	tk.MustExec("set @@session.tidb_analyze_skip_column_types = 'json, text, blob'") // text is not default.
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/statistics/handle/syncload/handleOneItemTaskPanic", `panic`))
+	tk.MustQuery("trace plan select * from t where content ='ab'")
+	require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/statistics/handle/syncload/handleOneItemTaskPanic"))
+}
+
 func TestConcurrentLoadHist(t *testing.T) {
 	store, dom := testkit.CreateMockStoreAndDomain(t)
 

pkg/statistics/handle/syncload/BUILD.bazel

@@ -0,0 +1,54 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "syncload",
+    srcs = ["stats_syncload.go"],
+    importpath = "github.com/pingcap/tidb/pkg/statistics/handle/syncload",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/config",
+        "//pkg/infoschema",
+        "//pkg/kv",
+        "//pkg/meta/model",
+        "//pkg/metrics",
+        "//pkg/parser/mysql",
+        "//pkg/sessionctx",
+        "//pkg/sessionctx/stmtctx",
+        "//pkg/sessionctx/variable",
+        "//pkg/statistics",
+        "//pkg/statistics/handle/storage",
+        "//pkg/statistics/handle/types",
+        "//pkg/table",
+        "//pkg/types",
+        "//pkg/util",
+        "//pkg/util/intest",
+        "//pkg/util/logutil",
+        "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@org_golang_x_sync//singleflight",
+        "@org_uber_go_zap//:zap",
+    ],
+)
+
+go_test(
+    name = "syncload_test",
+    timeout = "short",
+    srcs = ["stats_syncload_test.go"],
+    flaky = True,
+    race = "on",
+    shard_count = 6,
+    deps = [
+        ":syncload",
+        "//pkg/config",
+        "//pkg/meta/model",
+        "//pkg/parser/model",
+        "//pkg/sessionctx",
+        "//pkg/sessionctx/stmtctx",
+        "//pkg/statistics/handle/types",
+        "//pkg/testkit",
+        "//pkg/testkit/analyzehelper",
+        "//pkg/util/mathutil",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_stretchr_testify//require",
+    ],
+)