parser: also reduce literal list with charset

Signed-off-by: xhe <[email protected]>

Author: xhebox

pkg/parser/digester.go

@@ -364,8 +364,16 @@ func (d *sqlDigester) reduceLit(currTok *token, redact string, forBinding bool,
 		return
 	}
 
-	// Aggressive reduce lists.
+	// "_charset ?, _charset ?," => "..."
 	last4 := d.tokens.back(4)
+	if toPop := d.isGenericListWithCharset(last4); toPop != 0 {
+		d.tokens.popBack(toPop)
+		currTok.tok = genericSymbolList
+		currTok.lit = "..."
+		return
+	}
+
+	// Aggressive reduce lists.
 	if d.isGenericLists(last4) {
 		d.tokens.popBack(4)
 		currTok.tok = genericSymbolList
@@ -543,6 +551,27 @@ func (d *sqlDigester) isGenericList(last2 []token) (generic bool) {
 	return
 }
 
+func (d *sqlDigester) isGenericListWithCharset(last []token) int {
+	if len(last) < 3 {
+		return 0
+	}
+	toPop := 0
+	if len(last) >= 4 {
+		// elminate the first _charset
+		if last[0].tok == underscoreCS {
+			toPop = 1
+		}
+		last = last[1:]
+	}
+	if last[2].tok != underscoreCS {
+		return 0
+	}
+	if !d.isGenericList(last[:2]) {
+		return 0
+	}
+	return toPop + 3
+}
+
 func (d *sqlDigester) isOrderOrGroupBy() (orderOrGroupBy bool) {
 	var (
 		last []token

pkg/parser/digester_test.go

@@ -30,6 +30,9 @@ func TestNormalize(t *testing.T) {
 	}{
 		// Generic normalization rules
 		{"select _utf8mb4'123'", "select (_charset) ?"},
+		{"select * from b where id in (_utf8mb4'123')", "select * from `b` where `id` in ( (_charset) ? )"},
+		{"select * from b where id in (_utf8mb4'123', _binary'34')", "select * from `b` where `id` in ( ... )"},
+		{"select * from b where id in (_utf8mb4'123', _binary'34', _binary'56')", "select * from `b` where `id` in ( ... )"},
 		{"SELECT 1", "select ?"},
 		{"select null", "select ?"},
 		{"select \\N", "select ?"},