Merge branch 'release-3.1' into cherry-pick-2675-to-release-3.1

xuanyu66 · web-flow · commit 840f6b533629 · 2023-04-25T17:48:46.000+08:00
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -60,8 +60,11 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+#    - name: Autobuild
+#      uses: github/codeql-action/autobuild@v1
+    - name: build
+      run: |
+          mvn clean package -Dmaven.test.skip=true -B
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/core-test/pom.xml b/core-test/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala b/core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala
@@ -85,4 +85,7 @@ object TiConfigConst {
   val DEFAULT_GC_MAX_WAIT_TIME: Long = 24 * 60 * 60
   val DEFAULT_GC_SAFE_POINT_TTL: Int = 5 * 60
 
+  // cache load
+  val LOAD_TABLES: String = "spark.tispark.load_tables"
+  val DEFAULT_LOAD_TABLES: Boolean = true
 }
diff --git a/core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala b/core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala
@@ -237,6 +237,8 @@ object TiUtil {
       tiConf.setNewCollationEnable(conf.get(TiConfigConst.NEW_COLLATION_ENABLE).toBoolean)
     }
 
+    tiConf.setLoadTables(
+      conf.get(TiConfigConst.LOAD_TABLES, TiConfigConst.DEFAULT_LOAD_TABLES.toString).toBoolean)
     tiConf
   }
 
diff --git a/db-random-test/pom.xml b/db-random-test/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/docs/userguide_3.0.md b/docs/userguide_3.0.md
@@ -277,6 +277,7 @@ spark.sql("select t1.id,t2.id from spark_catalog.default.t t1 left join tidb_cat
 | `spark.tispark.replica_read.label`             | ""               | Only select TiKV store match specified labels. Format: label_x=value_x,label_y=value_y                                                                                                                                                                                                                                                                                                                                                            |
 | `spark.tispark.replica_read.address_whitelist` | ""               | Only select TiKV store with given ip addresses. Split mutil addresses by `,`                                                                                                                                                                                                                                                                                                                                                                      |
 | `spark.tispark.replica_read.address_blacklist` | ""               | Do not select TiKV store with given ip addresses. Split mutil addresses by `,`                                                                                                                                                                                                                                                                                                                                                                    |
+| `spark.tispark.load_tables`                     | true             | (experimental) Whether load all tables when we reload catalog cache. Disable it may cause table not find in scenarios where the table changes frequently.                                                                                                                                                                                                                                                                                           |
 
 ### TLS Configuration
 
@@ -385,13 +386,15 @@ TiSpark reads the range and hash partition table from TiDB.
 
 Currently, TiSpark doesn't support a MySQL/TiDB partition table syntax `select col_name from table_name partition(partition_name)`, but you can still use `where` condition to filter the partitions.
 
-TiSpark decides whether to apply partition pruning according to the partition type and the partition expression associated with the table. Currently, TiSpark partially apply partition pruning on range partition.
+TiSpark decides whether to apply partition pruning according to the partition type and the partition expression associated with the table.
+
+Currently, TiSpark partially apply partition pruning on range partition.
 
 The partition pruning is applied when the partition expression of the range partition is one of the following:
 
 + column expression
-+ `YEAR(col)` and its type is datetime/string/date literal that can be parsed as datetime.
-+ `TO_DAYS(col)` and its type is datetime/string/date literal that can be parsed as datetime.
++ `YEAR($argument)` where the argument is a column and its type is datetime or string literal
+  that can be parsed as datetime.
 
 If partition pruning is not applied, TiSpark's reading is equivalent to doing a table scan over all partitions.
 
diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
 
     <groupId>com.pingcap.tispark</groupId>
     <artifactId>tispark-parent</artifactId>
-    <version>3.1.2</version>
+    <version>3.1.3</version>
     <packaging>pom</packaging>
     <name>TiSpark Project Parent POM</name>
     <url>http://github.copm/pingcap/tispark</url>
diff --git a/spark-wrapper/spark-3.0/pom.xml b/spark-wrapper/spark-3.0/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/spark-wrapper/spark-3.1/pom.xml b/spark-wrapper/spark-3.1/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/spark-wrapper/spark-3.2/pom.xml b/spark-wrapper/spark-3.2/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/spark-wrapper/spark-3.3/pom.xml b/spark-wrapper/spark-3.3/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>com.pingcap.tispark</groupId>
     <artifactId>tispark-parent</artifactId>
-    <version>3.1.2</version>
+    <version>3.1.3</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/tikv-client/pom.xml b/tikv-client/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.pingcap.tispark</groupId>
         <artifactId>tispark-parent</artifactId>
-        <version>3.1.2</version>
+        <version>3.1.3</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java b/tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java
@@ -48,6 +48,12 @@ public class TiConfiguration implements Serializable {
   private static final boolean DEF_IGNORE_TRUNCATE = true;
   private static final boolean DEF_TRUNCATE_AS_WARNING = false;
   private static final int DEF_MAX_FRAME_SIZE = 2147483647; // 2 GB
+  private boolean loadTables = true;
+
+  public boolean getLoadTables() {
+    return loadTables;
+  }
+
   private static final int DEF_INDEX_SCAN_BATCH_SIZE = 20000;
   private static final int DEF_REGION_SCAN_DOWNGRADE_THRESHOLD = 10000000;
   // if keyRange size per request exceeds this limit, the request might be too large to be accepted
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/TiSession.java b/tikv-client/src/main/java/com/pingcap/tikv/TiSession.java
@@ -197,9 +197,11 @@ public synchronized Catalog getOrCreateSnapShotCatalog(TiTimestamp ts) {
     if (snapshotCatalog == null) {
       snapshotCatalog =
           new Catalog(
-              this::createSnapshotWithSnapshotTimestamp, conf.isShowRowId(), conf.getDBPrefix());
+              this::createSnapshotWithSnapshotTimestamp,
+              conf.isShowRowId(),
+              conf.getDBPrefix(),
+              conf.getLoadTables());
     }
-    snapshotCatalog.reloadCache(true);
     return snapshotCatalog;
   }
 
@@ -208,7 +210,12 @@ public Catalog getCatalog() {
     if (res == null) {
       synchronized (this) {
         if (catalog == null) {
-          catalog = new Catalog(this::createSnapshot, conf.isShowRowId(), conf.getDBPrefix());
+          catalog =
+              new Catalog(
+                  this::createSnapshot,
+                  conf.isShowRowId(),
+                  conf.getDBPrefix(),
+                  conf.getLoadTables());
         }
         res = catalog;
       }
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java b/tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java
@@ -41,13 +41,16 @@ public class Catalog implements AutoCloseable {
   private final Supplier<Snapshot> snapshotProvider;
   private CatalogCache metaCache;
   private static final AtomicLong lastUpdateTime = new AtomicLong(0);
+  private final boolean loadTables;
 
-  public Catalog(Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix) {
+  public Catalog(
+      Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix, boolean loadTables) {
     this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null");
     this.showRowId = showRowId;
     this.dbPrefix = dbPrefix;
-    metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false);
-    reloadCache(true);
+    this.loadTables = loadTables;
+    metaCache =
+        new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, loadTables);
   }
 
   @Override
@@ -67,18 +70,14 @@ public void reloadCache(boolean loadTables) {
     }
   }
 
-  private void reloadCache() {
-    reloadCache(false);
-  }
-
   public List<TiDBInfo> listDatabases() {
-    reloadCache();
+    reloadCache(false);
     return metaCache.listDatabases();
   }
 
   public List<TiTableInfo> listTables(TiDBInfo database) {
     Objects.requireNonNull(database, "database is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     if (showRowId) {
       return metaCache
           .listTables(database)
@@ -133,7 +132,7 @@ public TiTableInfo getTableFromCache(TiDBInfo database, String tableName) {
 
   public TiDBInfo getDatabase(String dbName) {
     Objects.requireNonNull(dbName, "dbName is null");
-    reloadCache();
+    reloadCache(false);
     return metaCache.getDatabase(dbName);
   }
 
@@ -148,7 +147,7 @@ public TiTableInfo getTable(String dbName, String tableName) {
   public TiTableInfo getTable(TiDBInfo database, String tableName) {
     Objects.requireNonNull(database, "database is null");
     Objects.requireNonNull(tableName, "tableName is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     TiTableInfo table = metaCache.getTable(database, tableName);
     if (showRowId && table != null) {
       return table.copyTableWithRowId();
diff --git a/tikv-client/src/test/java/com/pingcap/tikv/catalog/CatalogTest.java b/tikv-client/src/test/java/com/pingcap/tikv/catalog/CatalogTest.java
@@ -70,8 +70,8 @@ public void listDatabasesTest() {
     helper.addDatabase(265, "other");
     helper.setSchemaVersion(667);
 
-    ReflectionWrapper wrapper = new ReflectionWrapper(cat);
-    wrapper.call("reloadCache");
+    ReflectionWrapper wrapper = new ReflectionWrapper(cat, boolean.class);
+    wrapper.call("reloadCache", false);
 
     dbs = cat.listDatabases();
     assertEquals(3, dbs.size());

Original file line number	Diff line number	Diff line change
`@@ -85,4 +85,7 @@ object TiConfigConst {`
`85`	`85`	`val DEFAULT_GC_MAX_WAIT_TIME: Long = 24 * 60 * 60`
`86`	`86`	`val DEFAULT_GC_SAFE_POINT_TTL: Int = 5 * 60`
`87`	`87`
	`88`	`+ // cache load`
	`89`	`+ val LOAD_TABLES: String = "spark.tispark.load_tables"`
	`90`	`+ val DEFAULT_LOAD_TABLES: Boolean = true`
`88`	`91`	`}`
Original file line number	Diff line number	Diff line change
`@@ -237,6 +237,8 @@ object TiUtil {`
`237`	`237`	`tiConf.setNewCollationEnable(conf.get(TiConfigConst.NEW_COLLATION_ENABLE).toBoolean)`
`238`	`238`	`}`
`239`	`239`
	`240`	`+ tiConf.setLoadTables(`
	`241`	`+ conf.get(TiConfigConst.LOAD_TABLES, TiConfigConst.DEFAULT_LOAD_TABLES.toString).toBoolean)`
`240`	`242`	`tiConf`
`241`	`243`	`}`
`242`	`244`