tpch: support add invert index during preparing

Signed-off-by: Lloyd-Pottiger <[email protected]>

Author: Lloyd-Pottiger

cmd/go-tpc/tpch.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/pingcap/go-tpc/pkg/util"
 	"github.com/pingcap/go-tpc/tpch"
 	"github.com/spf13/cobra"
@@ -29,8 +30,13 @@ var queryTuningVars = []struct {
 
 // isSysVarSupported determines if a system variable is supported in given TiDB version
 // TODO: Every known sys var should have a minimal supported version and be checked individually. For now we just assume all sys vars are supported since 7.1.0.
-func isSysVarSupported(ver util.SemVersion, sysVar string) bool {
-	return ver.Compare(util.SemVersion{Major: 7, Minor: 1, Patch: 0}) >= 0
+func isSysVarSupported(ver *semver.Version, _ string) bool {
+	return ver.Compare(semver.Version{Major: 7, Minor: 1, Patch: 0}) >= 0
+}
+
+// isInvertedIndexSupported checks if inverted index is supported in given TiDB version
+func isInvertedIndexSupported(ver semver.Version) bool {
+	return ver.Compare(semver.Version{Major: 9, Minor: 0, Patch: 0, PreRelease: "beta.1"}) > 0
 }
 
 func executeTpch(action string) {
@@ -45,23 +51,37 @@ func executeTpch(action string) {
 		runtime.GOMAXPROCS(maxProcs)
 	}
 
-	if action == "run" && driver == mysqlDriver && tpchConfig.EnableQueryTuning {
+	isTiDB := false
+	var TiDBSemVer *semver.Version
+	if driver == mysqlDriver {
 		serverVer, err := getServerVersion(globalDB)
 		if err != nil {
 			panic(fmt.Errorf("get server version failed: %v", err))
 		}
 		fmt.Printf("Server version: %s\n", serverVer)
+		TiDBSemVer, isTiDB = util.NewTiDBSemVersion(serverVer)
+	}
 
-		if semVer, ok := util.NewTiDBSemVersion(serverVer); ok {
-			fmt.Printf("Enabling query tuning for TiDB version %s.\n", semVer.String())
-			if err := setTiDBQueryTuningVars(globalDB, semVer); err != nil {
+	if action == "run" && tpchConfig.EnableQueryTuning {
+		if isTiDB {
+			fmt.Printf("Enabling query tuning for TiDB version %s.\n", TiDBSemVer.String())
+			if err := setTiDBQueryTuningVars(globalDB, TiDBSemVer); err != nil {
 				panic(fmt.Errorf("set session variables failed: %v", err))
 			}
 		} else {
 			fmt.Printf("Query tuning is enabled(by default) but server version doesn't appear to be TiDB, skipping tuning.\n")
 		}
 	}
 
+	if tpchConfig.AddInvertedIndex {
+		if !isTiDB || !isInvertedIndexSupported(*TiDBSemVer) || tpchConfig.TiFlashReplica == 0 {
+			fmt.Printf("Inverted index is only supported when TiDB version is > 9.0.0-beta.1 and TiFlash replica > 0.\n")
+			tpchConfig.AddInvertedIndex = false
+		} else {
+			tpchConfig.AddInvertedIndex = true
+		}
+	}
+
 	tpchConfig.PlanReplayerConfig.Host = hosts[0]
 	tpchConfig.PlanReplayerConfig.StatusPort = statusPort
 
@@ -85,7 +105,7 @@ func getServerVersion(db *sql.DB) (string, error) {
 	return version, err
 }
 
-func setTiDBQueryTuningVars(db *sql.DB, ver util.SemVersion) error {
+func setTiDBQueryTuningVars(db *sql.DB, ver *semver.Version) error {
 	for _, v := range queryTuningVars {
 		if isSysVarSupported(ver, v.name) {
 			if _, err := db.Exec(fmt.Sprintf("SET SESSION %s = %s", v.name, v.value)); err != nil {
@@ -189,6 +209,10 @@ func registerTpch(root *cobra.Command) {
 		"enable-query-tuning",
 		true,
 		"Tune queries by setting some session variables known effective for tpch")
+	cmdRun.PersistentFlags().BoolVar(&tpchConfig.AddInvertedIndex,
+		"add-inverted-index",
+		true,
+		"Add some inverted index to accelerate queries")
 
 	var cmdCleanup = &cobra.Command{
 		Use:   "cleanup",

go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.0.0
+	github.com/coreos/go-semver v0.3.1
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/lib/pq v1.10.6

go.sum

@@ -46,6 +46,8 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
+github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=

pkg/util/version.go

@@ -3,76 +3,51 @@ package util
 import (
 	"strconv"
 	"strings"
-)
 
-type SemVersion struct {
-	Major int
-	Minor int
-	Patch int
-}
+	"github.com/coreos/go-semver/semver"
+)
 
 // @version is the `SELECT VERSION()` output of TiDB
-func NewTiDBSemVersion(version string) (SemVersion, bool) {
+func NewTiDBSemVersion(version string) (*semver.Version, bool) {
 	isTiDB := strings.Contains(strings.ToLower(version), "tidb")
 	if !isTiDB {
-		return SemVersion{}, false
+		return nil, false
 	}
 
 	verItems := strings.Split(version, "-v")
 	if len(verItems) < 2 {
-		return SemVersion{}, false
+		return nil, false
+	}
+	verParts := strings.Split(verItems[1], "-")
+	verStr := verParts[0]
+	var preReleaseStr string
+	if len(verParts) > 1 {
+		preReleaseStr = verParts[1]
 	}
-	verStr := strings.Split(verItems[1], "-")[0]
 
 	parts := strings.Split(verStr, ".")
 	if len(parts) < 3 {
-		return SemVersion{}, false
+		return nil, false
 	}
 
-	major, err := strconv.Atoi(parts[0])
+	major, err := strconv.ParseInt(parts[0], 10, 64)
 	if err != nil {
-		return SemVersion{}, false
+		return nil, false
 	}
-	minor, err := strconv.Atoi(parts[1])
+	minor, err := strconv.ParseInt(parts[1], 10, 64)
 	if err != nil {
-		return SemVersion{}, false
+		return nil, false
 	}
 
-	patch, err := strconv.Atoi(parts[2])
+	patch, err := strconv.ParseInt(parts[2], 10, 64)
 	if err != nil {
-		return SemVersion{}, false
+		return nil, false
 	}
 
-	return SemVersion{
-		Major: major,
-		Minor: minor,
-		Patch: patch,
+	return &semver.Version{
+		Major:      major,
+		Minor:      minor,
+		Patch:      patch,
+		PreRelease: semver.PreRelease(preReleaseStr),
 	}, true
 }
-
-func (s SemVersion) String() string {
-	return strconv.Itoa(s.Major) + "." + strconv.Itoa(s.Minor) + "." + strconv.Itoa(s.Patch)
-}
-
-func (s SemVersion) Compare(other SemVersion) int {
-	signum := func(x int) int {
-		if x > 0 {
-			return 1
-		}
-		if x < 0 {
-			return -1
-		}
-		return 0
-	}
-
-	if diff := s.Major - other.Major; diff != 0 {
-		return signum(diff)
-	}
-	if diff := s.Minor - other.Minor; diff != 0 {
-		return signum(diff)
-	}
-	if diff := s.Patch - other.Patch; diff != 0 {
-		return signum(diff)
-	}
-	return 0
-}

pkg/util/version_test.go

@@ -3,62 +3,68 @@ package util
 import (
 	"testing"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewTiDBSemVersion(t *testing.T) {
 	testCases := []struct {
 		name     string
 		input    string
-		expected SemVersion
+		expected *semver.Version
 		ok       bool
 	}{
 		{
 			name:     "normal case with addition",
 			input:    "5.7.25-TiDB-v7.1.0-alpha",
-			expected: SemVersion{Major: 7, Minor: 1, Patch: 0},
+			expected: &semver.Version{Major: 7, Minor: 1, Patch: 0, PreRelease: "alpha"},
 			ok:       true,
 		},
 		{
 			name:     "version without addition",
 			input:    "5.7.25-TiDB-v7.4.1",
-			expected: SemVersion{Major: 7, Minor: 4, Patch: 1},
+			expected: &semver.Version{Major: 7, Minor: 4, Patch: 1, PreRelease: ""},
 			ok:       true,
 		},
 		{
 			name:     "multi-part addition",
 			input:    "5.7.25-TiDB-v6.5.3-beta.2",
-			expected: SemVersion{Major: 6, Minor: 5, Patch: 3},
+			expected: &semver.Version{Major: 6, Minor: 5, Patch: 3, PreRelease: "beta.2"},
 			ok:       true,
 		},
 		{
 			name:     "empty addition due to trailing hyphen",
 			input:    "5.7.25-TiDB-v7.1.0-",
-			expected: SemVersion{Major: 7, Minor: 1, Patch: 0},
+			expected: &semver.Version{Major: 7, Minor: 1, Patch: 0, PreRelease: ""},
+			ok:       true,
+		},
+		{
+			input:    "8.0.11-TiDB-v9.0.0-beta.1.pre-547-g4d34cac",
+			expected: &semver.Version{Major: 9, Minor: 0, Patch: 0, PreRelease: "beta.1.pre"},
 			ok:       true,
 		},
 		{
 			name:     "non-tidb database",
 			input:    "MySQL 8.0.35",
-			expected: SemVersion{},
+			expected: nil,
 			ok:       false,
 		},
 		{
 			name:     "missing version prefix",
 			input:    "TiDB-7.2.0",
-			expected: SemVersion{},
+			expected: nil,
 			ok:       false,
 		},
 		{
 			name:     "invalid patch version",
 			input:    "5.7.25-TiDB-v7.1.x",
-			expected: SemVersion{},
+			expected: nil,
 			ok:       false,
 		},
 		{
 			name:     "insufficient version parts",
 			input:    "5.7.25-TiDB-v7.1",
-			expected: SemVersion{},
+			expected: nil,
 			ok:       false,
 		},
 	}
@@ -73,78 +79,3 @@ func TestNewTiDBSemVersion(t *testing.T) {
 		})
 	}
 }
-
-func TestSemVersionCompare(t *testing.T) {
-	testCases := []struct {
-		name     string
-		version1 SemVersion
-		version2 SemVersion
-		expected int
-	}{
-		{
-			name:     "major version greater",
-			version1: SemVersion{Major: 8, Minor: 0, Patch: 0},
-			version2: SemVersion{Major: 7, Minor: 5, Patch: 10},
-			expected: 1,
-		},
-		{
-			name:     "major version less",
-			version1: SemVersion{Major: 6, Minor: 9, Patch: 9},
-			version2: SemVersion{Major: 7, Minor: 0, Patch: 0},
-			expected: -1,
-		},
-		{
-			name:     "major same, minor greater",
-			version1: SemVersion{Major: 7, Minor: 2, Patch: 0},
-			version2: SemVersion{Major: 7, Minor: 1, Patch: 5},
-			expected: 1,
-		},
-		{
-			name:     "major same, minor less",
-			version1: SemVersion{Major: 7, Minor: 1, Patch: 10},
-			version2: SemVersion{Major: 7, Minor: 2, Patch: 0},
-			expected: -1,
-		},
-		{
-			name:     "major and minor same, patch greater",
-			version1: SemVersion{Major: 7, Minor: 1, Patch: 5},
-			version2: SemVersion{Major: 7, Minor: 1, Patch: 0},
-			expected: 1,
-		},
-		{
-			name:     "major and minor same, patch less",
-			version1: SemVersion{Major: 7, Minor: 1, Patch: 0},
-			version2: SemVersion{Major: 7, Minor: 1, Patch: 1},
-			expected: -1,
-		},
-		{
-			name:     "identical versions",
-			version1: SemVersion{Major: 7, Minor: 1, Patch: 0},
-			version2: SemVersion{Major: 7, Minor: 1, Patch: 0},
-			expected: 0,
-		},
-		{
-			name:     "extreme version differences",
-			version1: SemVersion{Major: 10, Minor: 0, Patch: 0},
-			version2: SemVersion{Major: 1, Minor: 99, Patch: 99},
-			expected: 1,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			result := tc.version1.Compare(tc.version2)
-			if result != tc.expected {
-				t.Errorf("Expected %v.Compare(%v) = %v, got %v",
-					tc.version1, tc.version2, tc.expected, result)
-			}
-
-			reverseResult := tc.version2.Compare(tc.version1)
-			expectedReverse := -tc.expected
-			if reverseResult != expectedReverse {
-				t.Errorf("Expected %v.Compare(%v) = %v, got %v",
-					tc.version2, tc.version1, expectedReverse, reverseResult)
-			}
-		})
-	}
-}